From cd978fd47679620cf88c5ef4563c413dc3989a56 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 18 Mar 2023 03:02:42 +0400 Subject: Add Python Version Tests Signed-off-by: Hassan Abouelela --- .pre-commit-config.yaml | 11 +++++++++++ scripts/set_versions.py | 38 +++++++++++++++++++++++++++++++------- snekbox/api/resources/eval.py | 5 +---- tests/api/test_eval.py | 39 +++++++++++++++++++++++++++++++++++++-- tests/test_integration.py | 30 +++++++++++++++++++++++++++++- 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bb6775..750a8d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,3 +44,14 @@ repos: --format, "::error file=%(path)s,line=%(row)d,col=%(col)d::[flake8] %(code)s: %(text)s", ] + - repo: local + hooks: + - id: python-version-script + name: check py versions + entry: python scripts/set_versions.py + language: system + always_run: true + pass_filenames: false + description: Check the Python versions around the project are up to date. If this fails, you most likely need to re-run the set_versions script. + args: + - --error-modified diff --git a/scripts/set_versions.py b/scripts/set_versions.py index 3bd2ab4..de90693 100644 --- a/scripts/set_versions.py +++ b/scripts/set_versions.py @@ -1,12 +1,24 @@ -"""Generate a Dockerfile from in.Dockerfile and a version JSON file, and write version info.""" +""" +Generate a Dockerfile from in.Dockerfile and a version JSON file, and write version info. +If the argument --error-modified is passed, the script will not write any output +and will exit with status code 1 if the output files were modified. +""" + +import difflib +import sys from pathlib import Path from textwrap import dedent -from scripts.python_version import ALL_VERSIONS, MAIN_VERSION +try: + from scripts.python_version import ALL_VERSIONS, MAIN_VERSION +except ModuleNotFoundError: + sys.path.insert(0, Path(__file__).parent.parent.absolute().as_posix()) + from scripts.python_version import ALL_VERSIONS, MAIN_VERSION DOCKERFILE_TEMPLATE = Path("scripts/in.Dockerfile").read_text("utf-8") DOCKERFILE = Path("Dockerfile") +ERROR_MODIFIED = "--error-modified" in sys.argv # Download and copy multiple python images into one layer python_build = "" @@ -32,12 +44,24 @@ for version in ALL_VERSIONS: python_build = f"FROM python:{MAIN_VERSION.image_tag} as base-first\n" + python_build # Write new dockerfile -DOCKERFILE.write_text( +# fmt: off +# Black makes the following block much less readable +dockerfile_out = ( "# THIS FILE IS AUTOGENERATED, DO NOT MODIFY! #\n" - + DOCKERFILE_TEMPLATE.replace("{python_install_commands}", python_build) - .replace("{final_base}", previous_layer) - .replace("{main_version_tag}", MAIN_VERSION.image_tag), - "utf-8", + + DOCKERFILE_TEMPLATE + .replace("{python_install_commands}", python_build) + .replace("{final_base}", previous_layer).replace("{main_version_tag}", MAIN_VERSION.image_tag) ) +# fmt: on + +if ERROR_MODIFIED: + if (original := DOCKERFILE.read_text("utf-8")) != dockerfile_out: + print("Dockerfile modified:") + print("\n".join(difflib.unified_diff(dockerfile_out.splitlines(), original.splitlines()))) + raise SystemExit(1) + else: + exit(0) +else: + DOCKERFILE.write_text(dockerfile_out, "utf-8") print("Finished!") diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py index c883ddb..2e6f243 100644 --- a/snekbox/api/resources/eval.py +++ b/snekbox/api/resources/eval.py @@ -30,10 +30,7 @@ class EvalResource: "properties": { "input": {"type": "string"}, "args": {"type": "array", "items": {"type": "string"}}, - "version": { - "type": "string", - "oneOf": [{"const": name} for name in VERSION_DISPLAY_NAMES], - }, + "version": {"enum": VERSION_DISPLAY_NAMES}, "files": { "type": "array", "items": { diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py index 37f90e7..44bd22b 100644 --- a/tests/api/test_eval.py +++ b/tests/api/test_eval.py @@ -1,5 +1,10 @@ +from unittest.mock import patch + from tests.api import SnekAPITestCase +from scripts.python_version import VERSION_DISPLAY_NAMES, Version +from snekbox.api.resources import eval + class TestEvalResource(SnekAPITestCase): PATH = "/eval" @@ -33,8 +38,16 @@ class TestEvalResource(SnekAPITestCase): self.assertEqual(expected, result.json) def test_post_invalid_data_400(self): - bodies = ({"args": 400}, {"args": [], "files": [215]}) - expects = ["400 is not of type 'array'", "215 is not of type 'object'"] + bodies = ( + {"args": 400}, + {"args": [], "files": [215]}, + {"input": "", "version": "random-gibberish"}, + ) + expects = [ + "400 is not of type 'array'", + "215 is not of type 'object'", + f"'random-gibberish' is not one of {VERSION_DISPLAY_NAMES}", + ] for body, expected in zip(bodies, expects): with self.subTest(): result = self.simulate_post(self.PATH, json=body) @@ -121,6 +134,28 @@ class TestEvalResource(SnekAPITestCase): self.assertEqual("Request data failed validation", result.json["title"]) self.assertIn("does not match", result.json["description"]) + def test_version_selection(self): + """A version argument in body properly configures the eval version.""" + # Configure ALL_VERSIONS to a well-known state to test the function regardless + # of version configuration + display_name = "Fake test name" + versions = [ + Version("tag1", "3.9", "pypy 3.9", False), + Version("tag2", "3.10", display_name, False), + Version("tag3", "3.11", "CPython 3.11", True), + ] + display_names = [version.display_name for version in versions] + + with ( + patch.object(eval, "ALL_VERSIONS", versions), + patch.object(eval, "VERSION_DISPLAY_NAMES", display_names), + patch.dict(eval.EvalResource.REQ_SCHEMA["properties"], version={"enum": display_names}), + ): + body = {"input": "", "version": display_name} + result = self.simulate_post(self.PATH, json=body) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json["version"], display_name) + def test_post_invalid_content_type_415(self): body = "{'input': 'foo'}" headers = {"Content-Type": "application/xml"} diff --git a/tests/test_integration.py b/tests/test_integration.py index aa21a2d..4a6e811 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -7,7 +7,7 @@ from textwrap import dedent from tests.gunicorn_utils import run_gunicorn -from scripts.python_version import MAIN_VERSION +from scripts.python_version import ALL_VERSIONS, MAIN_VERSION def b64encode_code(data: str): @@ -67,6 +67,34 @@ class IntegrationTests(unittest.TestCase): self.assertEqual(status, 200) self.assertEqual(json.loads(response)["stdout"], expected) + def test_eval_version(self): + """Test eval requests with specific python versions selected.""" + selected = None + for version in ALL_VERSIONS: + if not version.is_main: + selected = version + break + + if selected is None: + # Ideally we'd test with some non-main version to ensure the logic is correct + # but we're likely running under a configuration where a main version doesn't exist + # so we just make the best of it + selected = MAIN_VERSION + + with run_gunicorn(): + response, status = snekbox_request( + { + "input": "import sys; print(sys.version)", + "version": selected.display_name, + } + ) + parsed = json.loads(response) + + self.assertEqual(status, 200) + self.assertEqual(parsed["returncode"], 0) + self.assertEqual(parsed["version"], selected.display_name) + self.assertIn(selected.version_name, parsed["stdout"]) + def test_files_send_receive(self): """Test sending and receiving files to snekbox.""" with run_gunicorn(): -- cgit v1.2.3