Records Failed Unittests

Adds logic for saving submissions of failed unittests. Updates schema
docs.

Signed-off-by: Hassan Abouelela <[email protected]>

2 files changed, 33 insertions, 8 deletions

diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py
index c0a50f3..97cf2ac 100644
--- a/backend/routes/forms/submit.py
+++ b/backend/routes/forms/submit.py
@@ -168,16 +168,38 @@ class SubmitForm(Route):
             if any("unittests" in question.data for question in form.questions):
                 unittest_results = await execute_unittest(response_obj, form)
 
-                if not all(test.passed for test in unittest_results):
-                    # Return 500 if we encountered an internal error (code 99).
-                    status_code = 500 if any(
-                        test.return_code == 99 for test in unittest_results
-                    ) else 403
+                failures = []
+                status_code = 403
 
+                for test in unittest_results:
+                    response_obj.response[test.question_id] = {
+                        "value": response_obj.response[test.question_id],
+                        "passed": test.passed
+                    }
+
+                    if test.return_code == 0:
+                        test_names = [] if test.passed else test.result.split(";")
+                        response_obj.response[test.question_id]["failures"] = test_names
+
+                    # Report a failure on internal errors,
+                    # or if the test suite doesn't allow failures
+                    if not test.passed:
+                        allow_failure = (
+                            form.questions[test.question_index].data["unittests"]["allow_failure"]
+                        )
+
+                        if test.return_code == 99:
+                            failures.append(test)
+                            status_code = 500
+
+                        elif not allow_failure:
+                            failures.append(test)
+
+                if len(failures):
                     return JSONResponse({
                         "error": "failed_tests",
                         "test_results": [
-                            test._asdict() for test in unittest_results if not test.passed
+                            test._asdict() for test in failures
                         ]
                     }, status_code=status_code)
 
diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py
index 590cb52..c23ff48 100644
--- a/backend/routes/forms/unittesting.py
+++ b/backend/routes/forms/unittesting.py
@@ -7,13 +7,15 @@ import httpx
 from httpx import HTTPStatusError
 
 from backend.constants import SNEKBOX_URL
-from backend.models import FormResponse, Form
+from backend.models import Form, FormResponse
 
 with open("resources/unittest_template.py") as file:
     TEST_TEMPLATE = file.read()
 
 
-UnittestResult = namedtuple("UnittestResult", "question_id return_code passed result")
+UnittestResult = namedtuple(
+    "UnittestResult", "question_id question_index return_code passed result"
+)
 
 
 def filter_unittests(form: Form) -> Form:
@@ -119,6 +121,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit
 
             unittest_results.append(UnittestResult(
                 question_id=question.id,
+                question_index=index,
                 return_code=return_code,
                 passed=passed,
                 result=result