From b98da9ae3693b40427077f69092b4b8ee69d2543 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 04:06:00 +0300 Subject: Records Failed Unittests Adds logic for saving submissions of failed unittests. Updates schema docs. Signed-off-by: Hassan Abouelela --- backend/routes/forms/submit.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) (limited to 'backend/routes/forms/submit.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index c0a50f3..97cf2ac 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -168,16 +168,38 @@ class SubmitForm(Route): if any("unittests" in question.data for question in form.questions): unittest_results = await execute_unittest(response_obj, form) - if not all(test.passed for test in unittest_results): - # Return 500 if we encountered an internal error (code 99). - status_code = 500 if any( - test.return_code == 99 for test in unittest_results - ) else 403 + failures = [] + status_code = 403 + for test in unittest_results: + response_obj.response[test.question_id] = { + "value": response_obj.response[test.question_id], + "passed": test.passed + } + + if test.return_code == 0: + test_names = [] if test.passed else test.result.split(";") + response_obj.response[test.question_id]["failures"] = test_names + + # Report a failure on internal errors, + # or if the test suite doesn't allow failures + if not test.passed: + allow_failure = ( + form.questions[test.question_index].data["unittests"]["allow_failure"] + ) + + if test.return_code == 99: + failures.append(test) + status_code = 500 + + elif not allow_failure: + failures.append(test) + + if len(failures): return JSONResponse({ "error": "failed_tests", "test_results": [ - test._asdict() for test in unittest_results if not test.passed + test._asdict() for test in failures ] }, status_code=status_code) -- cgit v1.2.3 From 0ab266861a34feb576ff8872123b97a366d8359d Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 04:17:52 +0300 Subject: Reports All Unittest Failure Reasons Records the reason of failure for non-zero exit codes. Signed-off-by: Hassan Abouelela --- backend/routes/forms/submit.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'backend/routes/forms/submit.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 97cf2ac..fe2abd5 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -178,8 +178,15 @@ class SubmitForm(Route): } if test.return_code == 0: - test_names = [] if test.passed else test.result.split(";") - response_obj.response[test.question_id]["failures"] = test_names + failure_names = [] if test.passed else test.result.split(";") + elif test.return_code == 5: + failure_names = ["Could not parse user code."] + elif test.return_code == 6: + failure_names = ["Could not load user code."] + else: + failure_names = ["Internal error."] + + response_obj.response[test.question_id]["failures"] = failure_names # Report a failure on internal errors, # or if the test suite doesn't allow failures -- cgit v1.2.3 From 9b308d614f8ffeb29e21cd2a7fd7ff3400d94f16 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 13:14:50 +0300 Subject: Uses 422 For Failed Tests Uses 422 instead of 403 to indicate a test has failed. Co-authored-by: Joe Banks --- backend/routes/forms/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'backend/routes/forms/submit.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index fe2abd5..4691640 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -169,7 +169,7 @@ class SubmitForm(Route): unittest_results = await execute_unittest(response_obj, form) failures = [] - status_code = 403 + status_code = 422 for test in unittest_results: response_obj.response[test.question_id] = { -- cgit v1.2.3 From be7038458046b0973b47eca3c0b45860968c7cbd Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 2 Jun 2021 13:15:52 +0300 Subject: Documents Return Code Adds a comment which explains when a certain return_code is used. Co-authored-by: Joe Banks --- backend/routes/forms/submit.py | 1 + 1 file changed, 1 insertion(+) (limited to 'backend/routes/forms/submit.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 4691640..a6c3fbd 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -195,6 +195,7 @@ class SubmitForm(Route): form.questions[test.question_index].data["unittests"]["allow_failure"] ) + # An error while communicating with the test runner if test.return_code == 99: failures.append(test) status_code = 500 -- cgit v1.2.3 From 11f5f02d7f5ba7c9d1e4458e155ca9ce17d49024 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Thu, 3 Jun 2021 04:02:41 +0300 Subject: Verifies Unittest Error Responses Signed-off-by: Hassan Abouelela --- backend/routes/forms/submit.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'backend/routes/forms/submit.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index a6c3fbd..7229ee1 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -45,6 +45,18 @@ class PartialSubmission(BaseModel): captcha: Optional[str] +class UnittestError(BaseModel): + question_id: str + question_index: int + return_code: int + passed: bool + result: str + + +class UnittestErrorMessage(ErrorMessage): + test_results: list[UnittestError] + + class SubmitForm(Route): """ Submit a form with the provided form ID. @@ -58,7 +70,8 @@ class SubmitForm(Route): resp=Response( HTTP_200=SubmissionResponse, HTTP_404=ErrorMessage, - HTTP_400=ErrorMessage + HTTP_400=ErrorMessage, + HTTP_422=UnittestErrorMessage ), tags=["forms", "responses"] ) -- cgit v1.2.3