From cf2b8d3babf29552e5eeb1ab535506408e18a3e5 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 04:03:17 +0300 Subject: Adds Code Question Model Adds an explicit model for questions of type `code`, to allow more complex parsing. Updates schema docs. Signed-off-by: Hassan Abouelela --- backend/models/__init__.py | 3 ++- backend/models/question.py | 23 +++++++++++++++++++++++ backend/routes/forms/unittesting.py | 8 ++++---- 3 files changed, 29 insertions(+), 5 deletions(-) (limited to 'backend') diff --git a/backend/models/__init__.py b/backend/models/__init__.py index 29ccb24..8ad7f7f 100644 --- a/backend/models/__init__.py +++ b/backend/models/__init__.py @@ -2,13 +2,14 @@ from .antispam import AntiSpam from .discord_user import DiscordUser from .form import Form, FormList from .form_response import FormResponse, ResponseList -from .question import Question +from .question import CodeQuestion, Question __all__ = [ "AntiSpam", "DiscordUser", "Form", "FormResponse", + "CodeQuestion", "Question", "FormList", "ResponseList" diff --git a/backend/models/question.py b/backend/models/question.py index 7daeb5a..9829843 100644 --- a/backend/models/question.py +++ b/backend/models/question.py @@ -4,6 +4,25 @@ from pydantic import BaseModel, Field, root_validator, validator from backend.constants import QUESTION_TYPES, REQUIRED_QUESTION_TYPE_DATA +_TESTS_TYPE = t.Union[t.Dict[str, str], int] + + +class Unittests(BaseModel): + allow_failure: bool = False + tests: _TESTS_TYPE + + @validator("tests") + def validate_tests(cls, value: _TESTS_TYPE) -> _TESTS_TYPE: + if isinstance(value, dict) and not len(value.keys()): + raise ValueError("Must have at least one test in a test suite.") + + return value + + +class CodeQuestion(BaseModel): + language: str + unittests: t.Optional[Unittests] + class Question(BaseModel): """Schema model for form question.""" @@ -49,4 +68,8 @@ class Question(BaseModel): f"got {type(value['data'][key]).__name__} instead." ) + # Validate unittest options + if value.get("type").lower() == "code": + value["data"] = CodeQuestion(**value.get("data")).dict() + return value diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 3854314..590cb52 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -24,7 +24,7 @@ def filter_unittests(form: Form) -> Form: """ for question in form.questions: if question.type == "code" and "unittests" in question.data: - question.data["unittests"] = len(question.data["unittests"]) + question.data["unittests"]["tests"] = len(question.data["unittests"]["tests"]) return form @@ -62,7 +62,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit """Execute all the unittests in this form and return the results.""" unittest_results = [] - for question in form.questions: + for index, question in enumerate(form.questions): if question.type == "code" and "unittests" in question.data: passed = False @@ -70,12 +70,12 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit hidden_test_counter = count(1) hidden_tests = { test.lstrip("#").lstrip("test_"): next(hidden_test_counter) - for test in question.data["unittests"].keys() + for test in question.data["unittests"]["tests"].keys() if test.startswith("#") } # Compose runner code - unit_code = _make_unit_code(question.data["unittests"]) + unit_code = _make_unit_code(question.data["unittests"]["tests"]) user_code = _make_user_code(form_response.response[question.id]) code = TEST_TEMPLATE.replace("### USER CODE", user_code) -- cgit v1.2.3 From b98da9ae3693b40427077f69092b4b8ee69d2543 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 04:06:00 +0300 Subject: Records Failed Unittests Adds logic for saving submissions of failed unittests. Updates schema docs. Signed-off-by: Hassan Abouelela --- SCHEMA.md | 17 ++++++++++++++++- backend/routes/forms/submit.py | 34 ++++++++++++++++++++++++++++------ backend/routes/forms/unittesting.py | 7 +++++-- 3 files changed, 49 insertions(+), 9 deletions(-) (limited to 'backend') diff --git a/SCHEMA.md b/SCHEMA.md index 57c238a..93be830 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -169,10 +169,25 @@ Textareas require no additional configuration. | `_id`/`id` | MongoDB ObjectID | Random identifier used for the response | | `user` | Optional [user details object](#user-details-object) | An object describing the user that submitted if the form is not anonymous | | `antispam` | Optional [anti spam object](#anti-spam-object) | An object containing information about the anti-spam on the form submission | -| `response` | Object | Object containing question IDs mapping to the users answer | +| `response` | Object | Object containing question IDs mapping to the users answer* | | `form_id` | String | ID of the form that the user is submitting to | | `timestamp` | String | ISO formatted string of submission time. | + + * If the question is of type `code`, the response has the following structure: +```json +"response": { + "": { + "value": "", + "passed": bool, + "failures": ["", "", "", ...] + }, + ... +} +``` +* Values in `<>` are placeholders, while the rest are actual keys +* `passed` is True only if all tests in the suite passed. + ### User details object The user details contains the information returned by Discord alongside an `admin` boolean key representing that the user has admin privileges. The information returned from Discord can be found in the [Discord Developer portal](https://discord.com/developers/docs/resources/user#user-object). diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index c0a50f3..97cf2ac 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -168,16 +168,38 @@ class SubmitForm(Route): if any("unittests" in question.data for question in form.questions): unittest_results = await execute_unittest(response_obj, form) - if not all(test.passed for test in unittest_results): - # Return 500 if we encountered an internal error (code 99). - status_code = 500 if any( - test.return_code == 99 for test in unittest_results - ) else 403 + failures = [] + status_code = 403 + for test in unittest_results: + response_obj.response[test.question_id] = { + "value": response_obj.response[test.question_id], + "passed": test.passed + } + + if test.return_code == 0: + test_names = [] if test.passed else test.result.split(";") + response_obj.response[test.question_id]["failures"] = test_names + + # Report a failure on internal errors, + # or if the test suite doesn't allow failures + if not test.passed: + allow_failure = ( + form.questions[test.question_index].data["unittests"]["allow_failure"] + ) + + if test.return_code == 99: + failures.append(test) + status_code = 500 + + elif not allow_failure: + failures.append(test) + + if len(failures): return JSONResponse({ "error": "failed_tests", "test_results": [ - test._asdict() for test in unittest_results if not test.passed + test._asdict() for test in failures ] }, status_code=status_code) diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 590cb52..c23ff48 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -7,13 +7,15 @@ import httpx from httpx import HTTPStatusError from backend.constants import SNEKBOX_URL -from backend.models import FormResponse, Form +from backend.models import Form, FormResponse with open("resources/unittest_template.py") as file: TEST_TEMPLATE = file.read() -UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result") +UnittestResult = namedtuple( + "UnittestResult", "question_id question_index return_code passed result" +) def filter_unittests(form: Form) -> Form: @@ -119,6 +121,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit unittest_results.append(UnittestResult( question_id=question.id, + question_index=index, return_code=return_code, passed=passed, result=result -- cgit v1.2.3 From 0ab266861a34feb576ff8872123b97a366d8359d Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 04:17:52 +0300 Subject: Reports All Unittest Failure Reasons Records the reason of failure for non-zero exit codes. Signed-off-by: Hassan Abouelela --- backend/routes/forms/submit.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 97cf2ac..fe2abd5 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -178,8 +178,15 @@ class SubmitForm(Route): } if test.return_code == 0: - test_names = [] if test.passed else test.result.split(";") - response_obj.response[test.question_id]["failures"] = test_names + failure_names = [] if test.passed else test.result.split(";") + elif test.return_code == 5: + failure_names = ["Could not parse user code."] + elif test.return_code == 6: + failure_names = ["Could not load user code."] + else: + failure_names = ["Internal error."] + + response_obj.response[test.question_id]["failures"] = failure_names # Report a failure on internal errors, # or if the test suite doesn't allow failures -- cgit v1.2.3 From 9b308d614f8ffeb29e21cd2a7fd7ff3400d94f16 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 13:14:50 +0300 Subject: Uses 422 For Failed Tests Uses 422 instead of 403 to indicate a test has failed. Co-authored-by: Joe Banks --- backend/routes/forms/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index fe2abd5..4691640 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -169,7 +169,7 @@ class SubmitForm(Route): unittest_results = await execute_unittest(response_obj, form) failures = [] - status_code = 403 + status_code = 422 for test in unittest_results: response_obj.response[test.question_id] = { -- cgit v1.2.3 From be7038458046b0973b47eca3c0b45860968c7cbd Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 13:15:52 +0300 Subject: Documents Return Code Adds a comment which explains when a certain return_code is used. Co-authored-by: Joe Banks --- backend/routes/forms/submit.py | 1 + 1 file changed, 1 insertion(+) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 4691640..a6c3fbd 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -195,6 +195,7 @@ class SubmitForm(Route): form.questions[test.question_index].data["unittests"]["allow_failure"] ) + # An error while communicating with the test runner if test.return_code == 99: failures.append(test) status_code = 500 -- cgit v1.2.3 From ee15ceccc8b2b690d0c383ac596ca95c36180a9a Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 13:35:21 +0300 Subject: Adds Missing Docstrings Signed-off-by: Hassan Abouelela --- backend/models/question.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'backend') diff --git a/backend/models/question.py b/backend/models/question.py index 9829843..8972bc1 100644 --- a/backend/models/question.py +++ b/backend/models/question.py @@ -8,11 +8,13 @@ _TESTS_TYPE = t.Union[t.Dict[str, str], int] class Unittests(BaseModel): + """Schema model for unittest suites in code questions.""" allow_failure: bool = False tests: _TESTS_TYPE @validator("tests") def validate_tests(cls, value: _TESTS_TYPE) -> _TESTS_TYPE: + """Confirm that at least one test exists in a test suite.""" if isinstance(value, dict) and not len(value.keys()): raise ValueError("Must have at least one test in a test suite.") @@ -20,6 +22,7 @@ class Unittests(BaseModel): class CodeQuestion(BaseModel): + """Schema model for questions of type `code`.""" language: str unittests: t.Optional[Unittests] -- cgit v1.2.3 From 9991a8e8031ab6c1d33577bc759a6c7b03183ea3 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Thu, 3 Jun 2021 03:02:01 +0300 Subject: Clarify Validator Semantics Co-authored-by: Chris Signed-off-by: Hassan Abouelela --- backend/models/question.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/models/question.py b/backend/models/question.py index 8972bc1..5a1334a 100644 --- a/backend/models/question.py +++ b/backend/models/question.py @@ -15,7 +15,7 @@ class Unittests(BaseModel): @validator("tests") def validate_tests(cls, value: _TESTS_TYPE) -> _TESTS_TYPE: """Confirm that at least one test exists in a test suite.""" - if isinstance(value, dict) and not len(value.keys()): + if isinstance(value, dict) and len(value.keys()) == 0: raise ValueError("Must have at least one test in a test suite.") return value -- cgit v1.2.3 From 5e64af5a14fa6537b0e20db9931f27e9bc5e9c82 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Thu, 3 Jun 2021 03:32:21 +0300 Subject: Handles Code Questions With No Tests Adds a check to handle code questions with no test suites. Signed-off-by: Hassan Abouelela --- backend/routes/forms/unittesting.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index c23ff48..bd14edf 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -65,7 +65,19 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit unittest_results = [] for index, question in enumerate(form.questions): - if question.type == "code" and "unittests" in question.data: + if question.type == "code": + + # Exit early if the suite doesn't have any tests + if question.data["unittests"] is None: + unittest_results.append(UnittestResult( + question_id=question.id, + question_index=index, + return_code=0, + passed=True, + result="" + )) + continue + passed = False # Tests starting with an hashtag should have censored names. -- cgit v1.2.3 From 11f5f02d7f5ba7c9d1e4458e155ca9ce17d49024 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Thu, 3 Jun 2021 04:02:41 +0300 Subject: Verifies Unittest Error Responses Signed-off-by: Hassan Abouelela --- backend/routes/forms/submit.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index a6c3fbd..7229ee1 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -45,6 +45,18 @@ class PartialSubmission(BaseModel): captcha: Optional[str] +class UnittestError(BaseModel): + question_id: str + question_index: int + return_code: int + passed: bool + result: str + + +class UnittestErrorMessage(ErrorMessage): + test_results: list[UnittestError] + + class SubmitForm(Route): """ Submit a form with the provided form ID. @@ -58,7 +70,8 @@ class SubmitForm(Route): resp=Response( HTTP_200=SubmissionResponse, HTTP_404=ErrorMessage, - HTTP_400=ErrorMessage + HTTP_400=ErrorMessage, + HTTP_422=UnittestErrorMessage ), tags=["forms", "responses"] ) -- cgit v1.2.3 From d40a33b7329d7b99f8965b029665929f968c35ba Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 19 Jun 2021 20:35:13 +0300 Subject: Updates Unittest Filter To Match New Model Signed-off-by: Hassan Abouelela --- backend/routes/forms/unittesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index bd14edf..a830775 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -25,7 +25,7 @@ def filter_unittests(form: Form) -> Form: This is used to redact the exact tests when sending the form back to the frontend. """ for question in form.questions: - if question.type == "code" and "unittests" in question.data: + if question.type == "code" and question.data["unittests"] is not None: question.data["unittests"]["tests"] = len(question.data["unittests"]["tests"]) return form -- cgit v1.2.3