From c31bb72067d5192cbf8fb4ec523ee90ec32693d1 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 12:05:46 +0100 Subject: Add snekbox to the environment --- backend/constants.py | 1 + 1 file changed, 1 insertion(+) (limited to 'backend') diff --git a/backend/constants.py b/backend/constants.py index fedab64..cccf437 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -9,6 +9,7 @@ from enum import Enum # noqa FRONTEND_URL = os.getenv("FRONTEND_URL", "https://forms.pythondiscord.com") DATABASE_URL = os.getenv("DATABASE_URL") MONGO_DATABASE = os.getenv("MONGO_DATABASE", "pydis_forms") +SNEKBOX_URL = os.getenv("SNEKBOX_URL", "http://snekbox.default.svc.cluster.local/eval") OAUTH2_CLIENT_ID = os.getenv("OAUTH2_CLIENT_ID") OAUTH2_CLIENT_SECRET = os.getenv("OAUTH2_CLIENT_SECRET") -- cgit v1.2.3 From 6c38d1f153211e1731ed805da992fa5978ead91e Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 12:07:41 +0100 Subject: Support code unit testing through snekbox --- backend/routes/forms/unittesting.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 backend/routes/forms/unittesting.py (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py new file mode 100644 index 0000000..3e1d280 --- /dev/null +++ b/backend/routes/forms/unittesting.py @@ -0,0 +1,91 @@ +import ast +from collections import namedtuple +from textwrap import indent +from typing import Optional + +import httpx + +from backend.constants import SNEKBOX_URL +from backend.models import FormResponse, Form + +with open("resources/unittest_template.py") as file: + TEST_TEMPLATE = file.read() + + +UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result") + + +def _make_unit_code(units: dict[str, str]) -> str: + result = "" + + for unit_name, unit_code in units.items(): + result += f"\ndef test_{unit_name}(unit):\n{indent(unit_code, ' ')}" + + return indent(result, " ") + + +def _make_user_code(code: str) -> str: + # Make sure that we we escape triple quotes and backslashes in the user code + code = code.replace('"""', '\\"""').replace("\\", "\\\\") + return f'USER_CODE = """{code}"""' + + +async def _post_eval(code: str) -> Optional[dict[str, str]]: + data = {"input": code} + async with httpx.AsyncClient() as client: + response = await client.post(SNEKBOX_URL, json=data) + + if not response.status_code == 200: + return + + return response.json() + + +async def execute_unittest(form_response: FormResponse, form: Form) -> list[UnittestResult]: + unittest_results = [] + + for question in form.questions: + if question.type == "code" and "unittests" in question.data: + passed = False + + unit_code = _make_unit_code(question.data["unittests"]) + user_code = _make_user_code(form_response.response[question.id]) + + code = TEST_TEMPLATE.replace("### USER CODE", user_code).replace("### UNIT CODE", unit_code) + + # Make sure that the code is well formatted (we don't check for the user code) + try: + ast.parse(code) + except SyntaxError: + return_code = 99 + result = "Invalid generated unit code." + + else: + response = await _post_eval(code) + + if not response: + return_code = 99 + result = "Unable to contact code runner." + else: + return_code = int(response["returncode"]) + + if return_code not in (0, 5, 99): + return_code = 99 + result = "Internal error." + else: + stdout = response["stdout"] + passed = bool(int(stdout[0])) + + if not passed: + result = stdout[1:].strip() + else: + result = "" + + unittest_results.append(UnittestResult( + question_id=question.id, + return_code=return_code, + passed=passed, + result=result + )) + + return unittest_results -- cgit v1.2.3 From 96c659fce17a5aca17fb913cf587765bac90481f Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 12:08:11 +0100 Subject: Hook up unittesting in the submit protocol --- backend/routes/forms/submit.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index d8e6d35..85a4226 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -18,6 +18,7 @@ from starlette.responses import JSONResponse from backend.constants import FRONTEND_URL, FormFeatures, HCAPTCHA_API_SECRET from backend.models import Form, FormResponse from backend.route import Route +from backend.routes.forms.unittesting import execute_unittest from backend.validation import AuthorizationHeaders, ErrorMessage, api HCAPTCHA_VERIFY_URL = "https://hcaptcha.com/siteverify" @@ -127,6 +128,19 @@ class SubmitForm(Route): except ValidationError as e: return JSONResponse(e.errors(), status_code=422) + has_unittests = any("unittests" in question.data for question in form.questions) + if has_unittests: + unittest_results = await execute_unittest(response_obj, form) + + was_successful = all(test.passed for test in unittest_results) + if not was_successful: + status_code = 500 if any(test.return_code == 99 for test in unittest_results) else 200 + + return JSONResponse({ + "error": "failed_tests", + "test_results": [test._asdict() for test in unittest_results if not test.passed] + }, status_code=status_code) + await request.state.db.responses.insert_one( response_obj.dict(by_alias=True) ) -- cgit v1.2.3 From da6b581185e8bbe37e561a05827c8517824c7d2c Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 13:53:08 +0100 Subject: Switch to 100 chars line length and get rid of the noqas --- backend/constants.py | 8 ++++---- backend/models/form.py | 2 +- backend/routes/forms/form.py | 2 +- backend/routes/forms/submit.py | 15 +++++++++++---- backend/routes/forms/unittesting.py | 3 ++- tox.ini | 4 +++- 6 files changed, 22 insertions(+), 12 deletions(-) (limited to 'backend') diff --git a/backend/constants.py b/backend/constants.py index cccf437..59b56e0 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -1,9 +1,9 @@ from dotenv import load_dotenv -load_dotenv() +import os +import binascii +from enum import Enum -import os # noqa -import binascii # noqa -from enum import Enum # noqa +load_dotenv() FRONTEND_URL = os.getenv("FRONTEND_URL", "https://forms.pythondiscord.com") diff --git a/backend/models/form.py b/backend/models/form.py index 8e59905..eac0b63 100644 --- a/backend/models/form.py +++ b/backend/models/form.py @@ -47,7 +47,7 @@ class Form(BaseModel): if any(v not in allowed_values for v in value): raise ValueError("Form features list contains one or more invalid values.") - if FormFeatures.COLLECT_EMAIL in value and FormFeatures.REQUIRES_LOGIN not in value: # noqa + if FormFeatures.COLLECT_EMAIL in value and FormFeatures.REQUIRES_LOGIN not in value: raise ValueError("COLLECT_EMAIL feature require REQUIRES_LOGIN feature.") return value diff --git a/backend/routes/forms/form.py b/backend/routes/forms/form.py index b6b722e..e5f7ec6 100644 --- a/backend/routes/forms/form.py +++ b/backend/routes/forms/form.py @@ -26,7 +26,7 @@ class SingleForm(Route): @api.validate(resp=Response(HTTP_200=Form, HTTP_404=ErrorMessage), tags=["forms"]) async def get(self, request: Request) -> JSONResponse: """Returns single form information by ID.""" - admin = request.user.payload["admin"] if request.user.is_authenticated else False # noqa + admin = request.user.payload["admin"] if request.user.is_authenticated else False filters = { "_id": request.path_params["form_id"] diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 85a4226..c19fc2d 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -100,7 +100,10 @@ class SubmitForm(Route): if request.user.is_authenticated: response["user"] = request.user.payload - if FormFeatures.COLLECT_EMAIL.value in form.features and "email" not in response["user"]: # noqa + if ( + FormFeatures.COLLECT_EMAIL.value in form.features + and "email" not in response["user"] + ): return JSONResponse({ "error": "email_required" }, status_code=400) @@ -134,11 +137,15 @@ class SubmitForm(Route): was_successful = all(test.passed for test in unittest_results) if not was_successful: - status_code = 500 if any(test.return_code == 99 for test in unittest_results) else 200 + status_code = 500 if any( + test.return_code == 99 for test in unittest_results + ) else 200 return JSONResponse({ "error": "failed_tests", - "test_results": [test._asdict() for test in unittest_results if not test.passed] + "test_results": [ + test._asdict() for test in unittest_results if not test.passed + ] }, status_code=status_code) await request.state.db.responses.insert_one( @@ -186,7 +193,7 @@ class SubmitForm(Route): embed = { "title": "New Form Response", "description": f"{mention} submitted a response to `{form.name}`.", - "url": f"{FRONTEND_URL}/path_to_view_form/{response.id}", # noqa # TODO: Enter Form View URL + "url": f"{FRONTEND_URL}/path_to_view_form/{response.id}", # TODO: Enter Form View URL "timestamp": response.timestamp, "color": 7506394, } diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 3e1d280..fe8320f 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -51,7 +51,8 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit unit_code = _make_unit_code(question.data["unittests"]) user_code = _make_user_code(form_response.response[question.id]) - code = TEST_TEMPLATE.replace("### USER CODE", user_code).replace("### UNIT CODE", unit_code) + code = TEST_TEMPLATE.replace("### USER CODE", user_code) + code = code.replace("### UNIT CODE", unit_code) # Make sure that the code is well formatted (we don't check for the user code) try: diff --git a/tox.ini b/tox.ini index 48a3da6..afb3b34 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,10 @@ [flake8] -max-line-length=88 +max-line-length=100 exclude=.cache,.venv,.git docstring-convention=all import-order-style=pycharm ignore= # Type annotations ANN101,ANN102 + # Line breaks + W503 -- cgit v1.2.3 From d69f80e083ed1b9d91716609c7c063968aef22fa Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 14:30:39 +0100 Subject: Return 403 on failed tests --- backend/routes/forms/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index c19fc2d..7618a33 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -139,7 +139,7 @@ class SubmitForm(Route): if not was_successful: status_code = 500 if any( test.return_code == 99 for test in unittest_results - ) else 200 + ) else 403 return JSONResponse({ "error": "failed_tests", -- cgit v1.2.3 From 3acf8d85447f1d58c8b3d0d6997828f166dfac5f Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 14:46:44 +0100 Subject: Add support for hidden tests --- backend/routes/forms/unittesting.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index fe8320f..ddf0843 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -1,5 +1,6 @@ import ast from collections import namedtuple +from itertools import count from textwrap import indent from typing import Optional @@ -19,7 +20,7 @@ def _make_unit_code(units: dict[str, str]) -> str: result = "" for unit_name, unit_code in units.items(): - result += f"\ndef test_{unit_name}(unit):\n{indent(unit_code, ' ')}" + result += f"\ndef test_{unit_name.lstrip('#')}(unit):\n{indent(unit_code, ' ')}" return indent(result, " ") @@ -48,6 +49,13 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit if question.type == "code" and "unittests" in question.data: passed = False + hidden_test_counter = count(1) + hidden_tests = { + test.lstrip("#"): next(hidden_test_counter) + for test in question.data["unittests"].keys() + if test.startswith("#") + } + unit_code = _make_unit_code(question.data["unittests"]) user_code = _make_user_code(form_response.response[question.id]) @@ -78,7 +86,14 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit passed = bool(int(stdout[0])) if not passed: - result = stdout[1:].strip() + failed_tests = stdout[1:].strip().split(";") + + # Redact failed hidden tests + for i, failed_test in enumerate(failed_tests[:]): + if failed_test in hidden_tests: + failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" + + result = ";".join(failed_tests) else: result = "" -- cgit v1.2.3 From 6a1be658fd7fea03428f0ef1bbcce630ab290782 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 14:57:00 +0100 Subject: Censor unittests on GET /forms/$id --- backend/routes/forms/form.py | 3 +++ backend/routes/forms/unittesting.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) (limited to 'backend') diff --git a/backend/routes/forms/form.py b/backend/routes/forms/form.py index e5f7ec6..deb03ae 100644 --- a/backend/routes/forms/form.py +++ b/backend/routes/forms/form.py @@ -10,6 +10,7 @@ from starlette.responses import JSONResponse from backend.models import Form from backend.route import Route +from backend.routes.forms.unittesting import filter_unittests from backend.validation import ErrorMessage, OkayResponse, api @@ -37,6 +38,8 @@ class SingleForm(Route): if raw_form := await request.state.db.forms.find_one(filters): form = Form(**raw_form) + form = filter_unittests(form) + return JSONResponse(form.dict(admin=admin)) return JSONResponse({"error": "not_found"}, status_code=404) diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index ddf0843..0cb7d8d 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -16,6 +16,19 @@ with open("resources/unittest_template.py") as file: UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result") +def filter_unittests(form: Form) -> Form: + """ + Replace the unittest data section of code questions with the number of test cases. + + This is used to redact the exact tests when sending the form back to the frontend. + """ + for question in form.questions: + if question.type == "code" and "unittests" in question.data: + question.data["unittests"] = len(question.data["unittests"]) + + return form + + def _make_unit_code(units: dict[str, str]) -> str: result = "" -- cgit v1.2.3 From 0a9026dcdd23eaf7c48256eb7da5af774892673b Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 15:16:13 +0100 Subject: Document unittest code --- backend/routes/forms/submit.py | 2 ++ backend/routes/forms/unittesting.py | 20 ++++++++++++++------ resources/unittest_template.py | 13 +++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 7618a33..d6b549e 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -131,12 +131,14 @@ class SubmitForm(Route): except ValidationError as e: return JSONResponse(e.errors(), status_code=422) + # Run unittests if needed has_unittests = any("unittests" in question.data for question in form.questions) if has_unittests: unittest_results = await execute_unittest(response_obj, form) was_successful = all(test.passed for test in unittest_results) if not was_successful: + # Return 500 if we encountered an internal error (code 99). status_code = 500 if any( test.return_code == 99 for test in unittest_results ) else 403 diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 0cb7d8d..e038f3a 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -30,6 +30,7 @@ def filter_unittests(form: Form) -> Form: def _make_unit_code(units: dict[str, str]) -> str: + """Compose a dict mapping unit names to their code into an actual class body.""" result = "" for unit_name, unit_code in units.items(): @@ -39,14 +40,16 @@ def _make_unit_code(units: dict[str, str]) -> str: def _make_user_code(code: str) -> str: - # Make sure that we we escape triple quotes and backslashes in the user code - code = code.replace('"""', '\\"""').replace("\\", "\\\\") - return f'USER_CODE = """{code}"""' + """Compose the user code into an actual string variable.""" + # Make sure that we we escape triple quotes in the user code + code = code.replace('"""', '\\"""') + return f'USER_CODE = r"""{code}"""' async def _post_eval(code: str) -> Optional[dict[str, str]]: - data = {"input": code} + """Post the eval to snekbox and return the response.""" async with httpx.AsyncClient() as client: + data = {"input": code} response = await client.post(SNEKBOX_URL, json=data) if not response.status_code == 200: @@ -56,12 +59,14 @@ async def _post_eval(code: str) -> Optional[dict[str, str]]: async def execute_unittest(form_response: FormResponse, form: Form) -> list[UnittestResult]: + """Execute all the unittests in this form and return the results.""" unittest_results = [] for question in form.questions: if question.type == "code" and "unittests" in question.data: passed = False + # Tests starting with an hashtag should have censored names. hidden_test_counter = count(1) hidden_tests = { test.lstrip("#"): next(hidden_test_counter) @@ -69,19 +74,20 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit if test.startswith("#") } + # Compose runner code unit_code = _make_unit_code(question.data["unittests"]) user_code = _make_user_code(form_response.response[question.id]) code = TEST_TEMPLATE.replace("### USER CODE", user_code) code = code.replace("### UNIT CODE", unit_code) - # Make sure that the code is well formatted (we don't check for the user code) + # Make sure that the code is well formatted (we don't check for the user code). try: ast.parse(code) except SyntaxError: return_code = 99 result = "Invalid generated unit code." - + # The runner is correctly formatted, we can run it. else: response = await _post_eval(code) @@ -91,6 +97,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit else: return_code = int(response["returncode"]) + # Another code has been returned by CPython because of another failure. if return_code not in (0, 5, 99): return_code = 99 result = "Internal error." @@ -98,6 +105,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit stdout = response["stdout"] passed = bool(int(stdout[0])) + # If the test failed, we have to populate the result string. if not passed: failed_tests = stdout[1:].strip().split(";") diff --git a/resources/unittest_template.py b/resources/unittest_template.py index c792944..4c9b0bb 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -1,4 +1,5 @@ # flake8: noqa +"""This template is used inside snekbox to evaluate and test user code.""" import ast import io import os @@ -23,27 +24,26 @@ DEVNULL = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None) RESULT = io.StringIO() ORIGINAL_STDOUT = sys.stdout +# stdout/err is patched in order to control what is outputted by the runner sys.stdout = DEVNULL sys.stderr = DEVNULL def _exit_sandbox(code: int) -> NoReturn: """ + Exit the sandbox by printing the result to the actual stdout and exit with the provided code. + Codes: - 0: Executed with success - 5: Syntax error while parsing user code - 99: Internal error """ - result_content = RESULT.getvalue() - - print( - f"{result_content}", - file=ORIGINAL_STDOUT - ) + print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") sys.exit(code) def _load_user_module() -> ModuleType: + """Load the user code into a new module and return it.""" try: ast.parse(USER_CODE, "") except SyntaxError: @@ -74,6 +74,7 @@ def _main() -> None: try: + # Load the user code as a global module variable module = _load_user_module() _main() except Exception: -- cgit v1.2.3 From d466b8016c9fb5a5f23731d83254b0b94cf02990 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 15:44:37 +0100 Subject: Properly handle return codes 5 and 99 --- backend/routes/forms/unittesting.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index e038f3a..f7f6072 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -102,21 +102,25 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit return_code = 99 result = "Internal error." else: - stdout = response["stdout"] - passed = bool(int(stdout[0])) - - # If the test failed, we have to populate the result string. - if not passed: - failed_tests = stdout[1:].strip().split(";") - - # Redact failed hidden tests - for i, failed_test in enumerate(failed_tests[:]): - if failed_test in hidden_tests: - failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" - - result = ";".join(failed_tests) + # Parse the stdout if the tests ran successfully + if return_code == 0: + stdout = response["stdout"] + passed = bool(int(stdout[0])) + + # If the test failed, we have to populate the result string. + if not passed: + failed_tests = stdout[1:].strip().split(";") + + # Redact failed hidden tests + for i, failed_test in enumerate(failed_tests[:]): + if failed_test in hidden_tests: + failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" + + result = ";".join(failed_tests) + else: + result = "" else: - result = "" + result = response["stdout"] unittest_results.append(UnittestResult( question_id=question.id, -- cgit v1.2.3 From 8939c8e127d49f9f534679d5ff9bdef907730e13 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:15:15 +0100 Subject: Add return code 6 for exceptions when loading module --- backend/routes/forms/unittesting.py | 2 +- resources/unittest_template.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index f7f6072..c00fc4c 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -98,7 +98,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit return_code = int(response["returncode"]) # Another code has been returned by CPython because of another failure. - if return_code not in (0, 5, 99): + if return_code not in (0, 5, 6, 99): return_code = 99 result = "Internal error." else: diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 755f7cc..02d3894 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -25,6 +25,7 @@ def _exit_sandbox(code: int) -> NoReturn: Codes: - 0: Executed with success - 5: Syntax error while parsing user code + - 6: Uncaught exception while loading user code - 99: Internal error """ print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") @@ -74,8 +75,12 @@ try: sys.stderr = DEVNULL # Load the user code as a global module variable - module = _load_user_module() + try: + module = _load_user_module() + except Exception: + RESULT.write("Uncaught exception while loading user code.") + _exit_sandbox(6) _main() except Exception: - print("Uncaught exception inside runner.", file=RESULT) - _exit_sandbox(99) \ No newline at end of file + RESULT.write("Uncaught exception inside runner.") + _exit_sandbox(99) -- cgit v1.2.3 From 7d34cb8563d8c01e5f5d1b038e0fbd507063e853 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:28:51 +0100 Subject: Add return code 7 for processes killed by NsJail --- backend/routes/forms/unittesting.py | 47 ++++++++++++++++++++----------------- resources/unittest_template.py | 2 ++ 2 files changed, 27 insertions(+), 22 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index c00fc4c..57bf5db 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -50,7 +50,7 @@ async def _post_eval(code: str) -> Optional[dict[str, str]]: """Post the eval to snekbox and return the response.""" async with httpx.AsyncClient() as client: data = {"input": code} - response = await client.post(SNEKBOX_URL, json=data) + response = await client.post(SNEKBOX_URL, json=data, timeout=10) if not response.status_code == 200: return @@ -97,30 +97,33 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit else: return_code = int(response["returncode"]) + # Parse the stdout if the tests ran successfully + if return_code == 0: + stdout = response["stdout"] + passed = bool(int(stdout[0])) + + # If the test failed, we have to populate the result string. + if not passed: + failed_tests = stdout[1:].strip().split(";") + + # Redact failed hidden tests + for i, failed_test in enumerate(failed_tests[:]): + if failed_test in hidden_tests: + failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" + + result = ";".join(failed_tests) + else: + result = "" + elif return_code in (5, 6, 99): + result = response["stdout"] + # Killed by NsJail + elif return_code == 137: + return_code = 7 + result = "Timed out or ran out of memory." # Another code has been returned by CPython because of another failure. - if return_code not in (0, 5, 6, 99): + else: return_code = 99 result = "Internal error." - else: - # Parse the stdout if the tests ran successfully - if return_code == 0: - stdout = response["stdout"] - passed = bool(int(stdout[0])) - - # If the test failed, we have to populate the result string. - if not passed: - failed_tests = stdout[1:].strip().split(";") - - # Redact failed hidden tests - for i, failed_test in enumerate(failed_tests[:]): - if failed_test in hidden_tests: - failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" - - result = ";".join(failed_tests) - else: - result = "" - else: - result = response["stdout"] unittest_results.append(UnittestResult( question_id=question.id, diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 02d3894..38e3be8 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -27,6 +27,8 @@ def _exit_sandbox(code: int) -> NoReturn: - 5: Syntax error while parsing user code - 6: Uncaught exception while loading user code - 99: Internal error + + 137 can also be generated by NsJail when killing the process. """ print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") sys.exit(code) -- cgit v1.2.3 From 99f9a0a940a91f2b9894ebf10b0359bba41d1856 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:36:59 +0100 Subject: Make use of .raise_for_status() Co-authored-by: Hassan Abouelela <47495861+HassanAbouelela@users.noreply.github.com> --- backend/routes/forms/unittesting.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 57bf5db..cc9f814 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -2,9 +2,9 @@ import ast from collections import namedtuple from itertools import count from textwrap import indent -from typing import Optional import httpx +from httpx import HTTPStatusError from backend.constants import SNEKBOX_URL from backend.models import FormResponse, Form @@ -46,15 +46,13 @@ def _make_user_code(code: str) -> str: return f'USER_CODE = r"""{code}"""' -async def _post_eval(code: str) -> Optional[dict[str, str]]: +async def _post_eval(code: str) -> dict[str, str]: """Post the eval to snekbox and return the response.""" async with httpx.AsyncClient() as client: data = {"input": code} response = await client.post(SNEKBOX_URL, json=data, timeout=10) - if not response.status_code == 200: - return - + response.raise_for_status() return response.json() @@ -89,9 +87,9 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit result = "Invalid generated unit code." # The runner is correctly formatted, we can run it. else: - response = await _post_eval(code) - - if not response: + try: + response = await _post_eval(code) + except HTTPStatusError: return_code = 99 result = "Unable to contact code runner." else: -- cgit v1.2.3 From 2bdcab13f2d25dee98ce4f6a04ef6baf69ce5898 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:41:40 +0100 Subject: Don't try to parse the composed code --- backend/routes/forms/unittesting.py | 72 +++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 40 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index cc9f814..198d950 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -78,50 +78,42 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit code = TEST_TEMPLATE.replace("### USER CODE", user_code) code = code.replace("### UNIT CODE", unit_code) - - # Make sure that the code is well formatted (we don't check for the user code). + try: - ast.parse(code) - except SyntaxError: + response = await _post_eval(code) + except HTTPStatusError: return_code = 99 - result = "Invalid generated unit code." - # The runner is correctly formatted, we can run it. + result = "Unable to contact code runner." else: - try: - response = await _post_eval(code) - except HTTPStatusError: - return_code = 99 - result = "Unable to contact code runner." - else: - return_code = int(response["returncode"]) - - # Parse the stdout if the tests ran successfully - if return_code == 0: - stdout = response["stdout"] - passed = bool(int(stdout[0])) - - # If the test failed, we have to populate the result string. - if not passed: - failed_tests = stdout[1:].strip().split(";") - - # Redact failed hidden tests - for i, failed_test in enumerate(failed_tests[:]): - if failed_test in hidden_tests: - failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" - - result = ";".join(failed_tests) - else: - result = "" - elif return_code in (5, 6, 99): - result = response["stdout"] - # Killed by NsJail - elif return_code == 137: - return_code = 7 - result = "Timed out or ran out of memory." - # Another code has been returned by CPython because of another failure. + return_code = int(response["returncode"]) + + # Parse the stdout if the tests ran successfully + if return_code == 0: + stdout = response["stdout"] + passed = bool(int(stdout[0])) + + # If the test failed, we have to populate the result string. + if not passed: + failed_tests = stdout[1:].strip().split(";") + + # Redact failed hidden tests + for i, failed_test in enumerate(failed_tests[:]): + if failed_test in hidden_tests: + failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" + + result = ";".join(failed_tests) else: - return_code = 99 - result = "Internal error." + result = "" + elif return_code in (5, 6, 99): + result = response["stdout"] + # Killed by NsJail + elif return_code == 137: + return_code = 7 + result = "Timed out or ran out of memory." + # Another code has been returned by CPython because of another failure. + else: + return_code = 99 + result = "Internal error." unittest_results.append(UnittestResult( question_id=question.id, -- cgit v1.2.3 From 52f12f4ab939b467c2ba88f5f83094fb1392baa2 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:42:43 +0100 Subject: Make use of list.copy() instead of [:] --- backend/routes/forms/unittesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 198d950..c11e4ad 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -78,7 +78,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit code = TEST_TEMPLATE.replace("### USER CODE", user_code) code = code.replace("### UNIT CODE", unit_code) - + try: response = await _post_eval(code) except HTTPStatusError: @@ -97,7 +97,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit failed_tests = stdout[1:].strip().split(";") # Redact failed hidden tests - for i, failed_test in enumerate(failed_tests[:]): + for i, failed_test in enumerate(failed_tests.copy()): if failed_test in hidden_tests: failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" -- cgit v1.2.3 From e46047da80d1141849e0ac755b83ef50f47bd53c Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:43:35 +0100 Subject: Only filter units if we aren't using an admin token --- backend/routes/forms/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/form.py b/backend/routes/forms/form.py index deb03ae..dd1c83f 100644 --- a/backend/routes/forms/form.py +++ b/backend/routes/forms/form.py @@ -38,7 +38,8 @@ class SingleForm(Route): if raw_form := await request.state.db.forms.find_one(filters): form = Form(**raw_form) - form = filter_unittests(form) + if not admin: + form = filter_unittests(form) return JSONResponse(form.dict(admin=admin)) -- cgit v1.2.3 From 06c01e78abcb0ab8713a3ad375218e98aab2882f Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:44:26 +0100 Subject: Remove unneeded temp variable --- backend/routes/forms/submit.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index d6b549e..b3a6afd 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -132,12 +132,10 @@ class SubmitForm(Route): return JSONResponse(e.errors(), status_code=422) # Run unittests if needed - has_unittests = any("unittests" in question.data for question in form.questions) - if has_unittests: + if any("unittests" in question.data for question in form.questions): unittest_results = await execute_unittest(response_obj, form) - was_successful = all(test.passed for test in unittest_results) - if not was_successful: + if not all(test.passed for test in unittest_results): # Return 500 if we encountered an internal error (code 99). status_code = 500 if any( test.return_code == 99 for test in unittest_results -- cgit v1.2.3 From bfc44e81cb0ea6b9997d8ca701b3f525ddcd50df Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:45:35 +0100 Subject: Make _make_unit_code more readable --- backend/routes/forms/unittesting.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index c11e4ad..ef86e7f 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -34,7 +34,10 @@ def _make_unit_code(units: dict[str, str]) -> str: result = "" for unit_name, unit_code in units.items(): - result += f"\ndef test_{unit_name.lstrip('#')}(unit):\n{indent(unit_code, ' ')}" + result += ( + f"\ndef test_{unit_name.lstrip('#')}(unit):" # Function definition + f"\n{indent(unit_code, ' ')}" # Unit code + ) return indent(result, " ") -- cgit v1.2.3 From e57b7ea1f5d93b8f9ebea825a742ed6ec5be1088 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 18:18:12 +0100 Subject: Remove unused import --- backend/routes/forms/unittesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index ef86e7f..4b362e1 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -1,4 +1,3 @@ -import ast from collections import namedtuple from itertools import count from textwrap import indent @@ -15,6 +14,9 @@ with open("resources/unittest_template.py") as file: UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result") +# Mapping of questions to their generated +_unit_cache: dict[str, str] = {} + def filter_unittests(form: Form) -> Form: """ @@ -35,7 +37,7 @@ def _make_unit_code(units: dict[str, str]) -> str: for unit_name, unit_code in units.items(): result += ( - f"\ndef test_{unit_name.lstrip('#')}(unit):" # Function definition + f"\ndef test_{unit_name.lstrip('#')}(unit):" # Function definition f"\n{indent(unit_code, ' ')}" # Unit code ) -- cgit v1.2.3 From a1a14d8a82bb7d2a9021bca5a2b8fcb3fbc4406a Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 18:20:46 +0100 Subject: Properly hadnle hidden tests starting with test_ --- backend/routes/forms/unittesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 4b362e1..175701f 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -72,7 +72,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit # Tests starting with an hashtag should have censored names. hidden_test_counter = count(1) hidden_tests = { - test.lstrip("#"): next(hidden_test_counter) + test.lstrip("#").lstrip("test_"): next(hidden_test_counter) for test in question.data["unittests"].keys() if test.startswith("#") } -- cgit v1.2.3 From 9d2c3794a4c95f6c63a7de64172bc35a68403a4c Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 26 Feb 2021 14:21:00 +0100 Subject: Use base64 encoded code snippets --- backend/routes/forms/unittesting.py | 8 ++++---- resources/unittest_template.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 175701f..b12cff2 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -1,3 +1,4 @@ +import base64 from collections import namedtuple from itertools import count from textwrap import indent @@ -45,10 +46,9 @@ def _make_unit_code(units: dict[str, str]) -> str: def _make_user_code(code: str) -> str: - """Compose the user code into an actual string variable.""" - # Make sure that we we escape triple quotes in the user code - code = code.replace('"""', '\\"""') - return f'USER_CODE = r"""{code}"""' + """Compose the user code into an actual base64-encoded string variable.""" + code = base64.b64encode(code.encode("utf8")).decode("utf8") + return f'USER_CODE = b"{code}"' async def _post_eval(code: str) -> dict[str, str]: diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 38e3be8..2410278 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -1,6 +1,7 @@ # flake8: noqa """This template is used inside snekbox to evaluate and test user code.""" import ast +import base64 import io import os import sys @@ -36,14 +37,15 @@ def _exit_sandbox(code: int) -> NoReturn: def _load_user_module() -> ModuleType: """Load the user code into a new module and return it.""" + code = base64.b64decode(USER_CODE).decode("utf8") try: - ast.parse(USER_CODE, "") + ast.parse(code, "") except SyntaxError: RESULT.write("".join(traceback.format_exception(*sys.exc_info(), limit=0))) _exit_sandbox(5) _module = ModuleType("module") - exec(USER_CODE, _module.__dict__) + exec(code, _module.__dict__) return _module -- cgit v1.2.3 From 0e4a95b584f20f22e61314483147e81b0dcb5354 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Sat, 27 Feb 2021 17:00:53 +0100 Subject: Obliterate the _unit_cache variable --- backend/routes/forms/unittesting.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index b12cff2..3854314 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -15,9 +15,6 @@ with open("resources/unittest_template.py") as file: UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result") -# Mapping of questions to their generated -_unit_cache: dict[str, str] = {} - def filter_unittests(form: Form) -> Form: """ -- cgit v1.2.3