aboutsummaryrefslogtreecommitdiffstats
path: root/resources/unittest_template.py
blob: 28dd588ba68dd1f0fa1512fc6b6059f90b8d3a22 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# flake8: noqa
"""This template is used inside snekbox to evaluate and test user code."""
import ast
import base64
import functools
import io
import sys
import traceback
import typing
import unittest
from itertools import chain
from types import ModuleType, SimpleNamespace
from typing import NoReturn


### USER CODE


class RunnerTestCase(unittest.IsolatedAsyncioTestCase):
### UNIT CODE


normal_exit = False
_EXIT_WRAPPER_TYPE = typing.Callable[[int], None]


def _exit_sandbox(code: int, stdout: io.StringIO, result_writer: io.StringIO) -> NoReturn:
    """
    Exit the sandbox by printing the result to the actual stdout and exit with the provided code.

    Codes:
    - 0: Executed with success
    - 5: Syntax error while parsing user code
    - 6: Uncaught exception while loading user code
    - 99: Internal error

    137 can also be generated by NsJail when killing the process.
    """
    print(result_writer.getvalue(), file=stdout, end="")
    global normal_exit
    normal_exit = True
    sys.exit(code)


def _load_user_module(result_writer, exit_wrapper: _EXIT_WRAPPER_TYPE) -> ModuleType:
    """Load the user code into a new module and return it."""
    code = base64.b64decode(USER_CODE).decode("utf8")
    try:
        ast.parse(code, "<input>")
    except SyntaxError:
        result_writer.write("".join(traceback.format_exception(*sys.exc_info(), limit=0)))
        exit_wrapper(5)

    _module = ModuleType("module")
    # It's necessary to manually add the module to the sys modules
    # Dataclasses do not add themselves which causes issues for us with this type of dynamic loading
    # if from __future__ import annotations is also used
    # See: https://github.com/mkdocs/mkdocs/issues/3141 and https://github.com/sqlalchemy/alembic/issues/1419
    sys.modules[_module.__name__] = _module

    exec(code, _module.__dict__)

    return _module


def _main(result_writer: io.StringIO, module: ModuleType, exit_wrapper: _EXIT_WRAPPER_TYPE) -> None:
    suite = unittest.defaultTestLoader.loadTestsFromTestCase(RunnerTestCase)
    globals()["module"] = module
    result = suite.run(unittest.TestResult())

    result_writer.write(str(int(result.wasSuccessful())))

    if not result.wasSuccessful():
        result_writer.write(
            ";".join(chain(
                (error[0]._testMethodName.removeprefix("test_") for error in result.errors),
                (failure[0]._testMethodName.removeprefix("test_") for failure in result.failures)
            ))
        )

    exit_wrapper(0)


def _entry():
    result_writer = io.StringIO()
    exit_wrapper = functools.partial(_exit_sandbox, stdout=sys.stdout, result_writer=result_writer)

    try:
        # Fake file object not writing anything
        devnull = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None)

        # stdout/err is patched in order to control what is outputted by the runner
        sys.__stdout__ = sys.stdout = devnull
        sys.__stderr__ = sys.stderr = devnull

        # Load the user code as a global module variable
        try:
            module = _load_user_module(result_writer, exit_wrapper)
        except BaseException as e:
            result_writer.write(f"Uncaught exception while loading user code: {e}")
            exit_wrapper(6)

        _main(result_writer, module, exit_wrapper)
    except BaseException as e:
        if isinstance(e, SystemExit) and normal_exit:
            raise e from None
        result_writer.write(f"Uncaught exception inside runner: {e}")
        exit_wrapper(99)

_entry()