From 063ed5585ba13e4629cf783f4fbe6a5a360744c1 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 14:34:17 -0700 Subject: Add a pyproject.toml --- .dockerignore | 1 + Makefile | 2 +- pyproject.toml | 44 +++++++++++++++++++++++++++++++++++++++++++ requirements/requirements.in | 7 ------- requirements/requirements.pip | 12 ++++++------ 5 files changed, 52 insertions(+), 14 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements/requirements.in diff --git a/.dockerignore b/.dockerignore index 30ecfd4..b2a96dd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ !requirements/ !tests !LICENSE +!pyproject.toml diff --git a/Makefile b/Makefile index 8a2705f..e4c8383 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ setup: install-piptools .PHONY: upgrade upgrade: install-piptools - $(PIP_COMPILE_CMD) -o requirements/requirements.pip requirements/requirements.in + $(PIP_COMPILE_CMD) -o requirements/requirements.pip pyproject.toml $(PIP_COMPILE_CMD) -o requirements/coverage.pip requirements/coverage.in $(PIP_COMPILE_CMD) -o requirements/coveralls.pip requirements/coveralls.in $(PIP_COMPILE_CMD) -o requirements/lint.pip requirements/lint.in diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..da921bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "snekbox" +version = "1.0.0" +description = "HTTP REST API for sanboxed execution of arbitrary Python code." +readme = "README.md" +license = {text = "MIT"} +authors = [{name = "Python Discord", email = "info@pythondiscord.com"}] +keywords = ["sandbox", "nsjail", "HTTP REST API"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + "Topic :: Security", + "Topic :: Software Development :: Interpreters", +] + +requires-python = ">=3.10" +dependencies = [ + # Sentry's Falcon integration relies on api_helpers (falconry/falcon#1902). + "falcon>=3.0.1", + "gunicorn>=20", + "jsonschema>=4.0", + "protobuf>=3.19", + "sentry-sdk[falcon]>=1.5.4" +] + +[project.urls] +source = "https://github.com/python-discord/snekbox" +tracker = "https://github.com/python-discord/snekbox/issues" + +[project.scripts] +snekbox = "snekbox.__main__:main" + +[tool.setuptools] +packages = ["snekbox"] diff --git a/requirements/requirements.in b/requirements/requirements.in deleted file mode 100644 index 775ad39..0000000 --- a/requirements/requirements.in +++ /dev/null @@ -1,7 +0,0 @@ -# Sentry's Falcon integration relies on api_helpers. See falconry/falcon#1902 -falcon>=3.0.1 - -gunicorn>=20 -jsonschema>=4.0 -protobuf>=3.19 -sentry-sdk[falcon]>=1.5.4 diff --git a/requirements/requirements.pip b/requirements/requirements.pip index 21b6678..d9587c1 100644 --- a/requirements/requirements.pip +++ b/requirements/requirements.pip @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --output-file=requirements/requirements.pip requirements/requirements.in +# pip-compile --output-file=requirements/requirements.pip pyproject.toml # attrs==21.4.0 # via jsonschema @@ -10,18 +10,18 @@ certifi==2022.5.18.1 # via sentry-sdk falcon==3.1.0 # via - # -r requirements/requirements.in # sentry-sdk + # snekbox (pyproject.toml) gunicorn==20.1.0 - # via -r requirements/requirements.in + # via snekbox (pyproject.toml) jsonschema==4.5.1 - # via -r requirements/requirements.in + # via snekbox (pyproject.toml) protobuf==4.21.1 - # via -r requirements/requirements.in + # via snekbox (pyproject.toml) pyrsistent==0.18.1 # via jsonschema sentry-sdk[falcon]==1.5.12 - # via -r requirements/requirements.in + # via snekbox (pyproject.toml) urllib3==1.26.9 # via sentry-sdk -- cgit v1.2.3 From f4ee29f9a1ad96796a07c1af301bf865f1bbdb00 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 14:34:57 -0700 Subject: Tweak Makefile format --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e4c8383..add4293 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,11 @@ lint: setup pre-commit run --all-files # Fix ownership of the coverage file even if tests fail & preserve exit code -# Install numpy because a test checks if it's importable .PHONY: test test: docker-compose build -q --force-rm docker-compose run --entrypoint /bin/bash --rm snekbox -c \ - 'coverage run -m unittest; e=$?; chown --reference=. .coverage; exit $e' + 'coverage run -m unittest; e=$?; chown --reference=. .coverage; exit $e' .PHONY: report report: setup -- cgit v1.2.3 From 0f7c541ba8896c314ef1932ff376ffdc7989eb00 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 14:53:40 -0700 Subject: Move coverage config into pyproject.toml --- .coveragerc | 13 ------------- pyproject.toml | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ce475d7..0000000 --- a/.coveragerc +++ /dev/null @@ -1,13 +0,0 @@ -[run] -branch = true -data_file = ${COVERAGE_DATAFILE-.coverage} -include = snekbox/* -omit = - snekbox/api/app.py - snekbox/config_pb2.py -relative_files = true - -[report] -exclude_lines = - pragma: no cover - if DEBUG diff --git a/pyproject.toml b/pyproject.toml index da921bb..09b69ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,3 +42,19 @@ snekbox = "snekbox.__main__:main" [tool.setuptools] packages = ["snekbox"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if DEBUG" +] + +[tool.coverage.run] +branch = true +data_file = "${COVERAGE_DATAFILE-.coverage}" +include = ["snekbox/*"] +omit = [ + "snekbox/api/app.py", + "snekbox/config_pb2.py" +] +relative_files = true -- cgit v1.2.3 From 8f62c90a96e3634c2d41cda96afefaff5c1fb54d Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 19:57:45 -0700 Subject: Move logging code to separate utility modules --- config/gunicorn.conf.py | 2 +- snekbox/__init__.py | 49 +++-------------------------------------------- snekbox/utils/__init__.py | 4 ++-- snekbox/utils/gunicorn.py | 33 +++++++++++++++++++++++++++++++ snekbox/utils/logging.py | 33 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 49 deletions(-) create mode 100644 snekbox/utils/gunicorn.py create mode 100644 snekbox/utils/logging.py diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 5ab11f4..5812c07 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -1,5 +1,5 @@ workers = 2 bind = "0.0.0.0:8060" -logger_class = "snekbox.GunicornLogger" +logger_class = "snekbox.utils.gunicorn.GunicornLogger" access_logformat = "%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss" access_logfile = "-" diff --git a/snekbox/__init__.py b/snekbox/__init__.py index 42628d4..1525ebd 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,51 +1,8 @@ -import logging import os -import sys -import sentry_sdk -from gunicorn import glogging -from gunicorn.config import Config -from sentry_sdk.integrations.falcon import FalconIntegration +from snekbox.utils.logging import init_logger, init_sentry DEBUG = os.environ.get("DEBUG", False) -GIT_SHA = os.environ.get("GIT_SHA", "development") -sentry_sdk.init( - dsn=os.environ.get("SNEKBOX_SENTRY_DSN", ""), - integrations=[FalconIntegration()], - send_default_pii=True, - release=f"snekbox@{GIT_SHA}" -) - - -class GunicornLogger(glogging.Logger): - """Logger for Gunicorn with custom formatting and support for the DEBUG environment variable.""" - - error_fmt = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" - access_fmt = error_fmt - datefmt = None # Use the default ISO 8601 format - - def setup(self, cfg: Config) -> None: - """ - Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set. - - Note: Access and syslog handlers would need to be recreated to use a custom date format - because they are created with an unspecified datefmt argument by default. - """ - super().setup(cfg) - - if DEBUG: - self.loglevel = logging.DEBUG - else: - self.loglevel = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO) - - self.error_log.setLevel(self.loglevel) - - -log = logging.getLogger("snekbox") -log.setLevel(logging.DEBUG if DEBUG else logging.INFO) -log.propagate = True -formatter = logging.Formatter(GunicornLogger.error_fmt) -handler = logging.StreamHandler(sys.stdout) -handler.setFormatter(formatter) -log.addHandler(handler) +init_sentry() +init_logger(DEBUG) diff --git a/snekbox/utils/__init__.py b/snekbox/utils/__init__.py index 5a7b632..6d6bc32 100644 --- a/snekbox/utils/__init__.py +++ b/snekbox/utils/__init__.py @@ -1,3 +1,3 @@ -from . import cgroup, swap +from . import cgroup, logging, swap -__all__ = ("cgroup", "swap") +__all__ = ("cgroup", "logging", "swap") diff --git a/snekbox/utils/gunicorn.py b/snekbox/utils/gunicorn.py new file mode 100644 index 0000000..68e3ed9 --- /dev/null +++ b/snekbox/utils/gunicorn.py @@ -0,0 +1,33 @@ +import logging + +from gunicorn import glogging +from gunicorn.config import Config + +from snekbox import DEBUG +from .logging import FORMAT + +__all__ = ("GunicornLogger",) + + +class GunicornLogger(glogging.Logger): + """Logger for Gunicorn with custom formatting and support for the DEBUG environment variable.""" + + error_fmt = FORMAT + access_fmt = error_fmt + datefmt = None # Use the default ISO 8601 format + + def setup(self, cfg: Config) -> None: + """ + Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set. + + Note: Access and syslog handlers would need to be recreated to use a custom date format + because they are created with an unspecified datefmt argument by default. + """ + super().setup(cfg) + + if DEBUG: + self.loglevel = logging.DEBUG + else: + self.loglevel = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO) + + self.error_log.setLevel(self.loglevel) diff --git a/snekbox/utils/logging.py b/snekbox/utils/logging.py new file mode 100644 index 0000000..e5afd0c --- /dev/null +++ b/snekbox/utils/logging.py @@ -0,0 +1,33 @@ +import logging +import os +import sys + +import sentry_sdk +from sentry_sdk.integrations.falcon import FalconIntegration + +__all__ = ("FORMAT", "init_logger", "init_sentry") + +FORMAT = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" + + +def init_logger(debug: bool) -> None: + """Initialise the root logger with a handler that outputs to stdout.""" + log = logging.getLogger("snekbox") + log.setLevel(logging.DEBUG if debug else logging.INFO) + log.propagate = True + + formatter = logging.Formatter(FORMAT) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + log.addHandler(handler) + + +def init_sentry() -> None: + """Initialise the Sentry SDK.""" + git_sha = os.environ.get("GIT_SHA", "development") + sentry_sdk.init( + dsn=os.environ.get("SNEKBOX_SENTRY_DSN", ""), + integrations=[FalconIntegration()], + send_default_pii=True, + release=f"snekbox@{git_sha}" + ) -- cgit v1.2.3 From 812374109a2adef13dd1a810cc69643c33de884f Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 20:01:47 -0700 Subject: Make Sentry SDK and gunicorn optional dependencies Falcon provides a WSGI app which can be used by any server, not just gunicorn. Thus, make gunicorn optional in case the user wants to use a different server. There shouldn't be any import errors since the class is now in an isolated module. The only time that module is imported is when gunicorn loads its config. Sentry is there for Python Discord mainly, so this dependency shouldn't be imposed on others. --- Makefile | 3 ++- pyproject.toml | 6 ++++-- requirements/requirements.pip | 2 +- snekbox/utils/logging.py | 11 +++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index add4293..a30579a 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,8 @@ setup: install-piptools .PHONY: upgrade upgrade: install-piptools - $(PIP_COMPILE_CMD) -o requirements/requirements.pip pyproject.toml + $(PIP_COMPILE_CMD) -o requirements/requirements.pip \ + --extra gunicorn --extra sentry pyproject.toml $(PIP_COMPILE_CMD) -o requirements/coverage.pip requirements/coverage.in $(PIP_COMPILE_CMD) -o requirements/coveralls.pip requirements/coveralls.in $(PIP_COMPILE_CMD) -o requirements/lint.pip requirements/lint.in diff --git a/pyproject.toml b/pyproject.toml index 09b69ac..abcf04f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,12 +27,14 @@ requires-python = ">=3.10" dependencies = [ # Sentry's Falcon integration relies on api_helpers (falconry/falcon#1902). "falcon>=3.0.1", - "gunicorn>=20", "jsonschema>=4.0", "protobuf>=3.19", - "sentry-sdk[falcon]>=1.5.4" ] +[project.optional-dependencies] +gunicorn = ["gunicorn>=20"] +sentry = ["sentry-sdk[falcon]>=1.5.4"] + [project.urls] source = "https://github.com/python-discord/snekbox" tracker = "https://github.com/python-discord/snekbox/issues" diff --git a/requirements/requirements.pip b/requirements/requirements.pip index d9587c1..034f104 100644 --- a/requirements/requirements.pip +++ b/requirements/requirements.pip @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --output-file=requirements/requirements.pip pyproject.toml +# pip-compile --extra=gunicorn --extra=sentry --output-file=requirements/requirements.pip pyproject.toml # attrs==21.4.0 # via jsonschema diff --git a/snekbox/utils/logging.py b/snekbox/utils/logging.py index e5afd0c..9a713f8 100644 --- a/snekbox/utils/logging.py +++ b/snekbox/utils/logging.py @@ -2,9 +2,6 @@ import logging import os import sys -import sentry_sdk -from sentry_sdk.integrations.falcon import FalconIntegration - __all__ = ("FORMAT", "init_logger", "init_sentry") FORMAT = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" @@ -23,7 +20,13 @@ def init_logger(debug: bool) -> None: def init_sentry() -> None: - """Initialise the Sentry SDK.""" + """Initialise the Sentry SDK if it's installed.""" + try: + import sentry_sdk + from sentry_sdk.integrations.falcon import FalconIntegration + except ImportError: + return + git_sha = os.environ.get("GIT_SHA", "development") sentry_sdk.init( dsn=os.environ.get("SNEKBOX_SENTRY_DSN", ""), -- cgit v1.2.3 From 409a76fa4aee3e2ee979f402cb4662cfd0a6aef4 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 20:52:26 -0700 Subject: Remove redundant module for creating the WSGI app --- Dockerfile | 2 +- snekbox/api/app.py | 3 --- tests/gunicorn_utils.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 snekbox/api/app.py diff --git a/Dockerfile b/Dockerfile index c207a2b..7331ab4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ COPY config/ /snekbox/config/ FROM venv ENTRYPOINT ["gunicorn"] -CMD ["-c", "config/gunicorn.conf.py", "snekbox.api.app"] +CMD ["-c", "config/gunicorn.conf.py", "snekbox.api:SnekAPI"] COPY . /snekbox WORKDIR /snekbox diff --git a/snekbox/api/app.py b/snekbox/api/app.py deleted file mode 100644 index c71e246..0000000 --- a/snekbox/api/app.py +++ /dev/null @@ -1,3 +0,0 @@ -from . import SnekAPI - -application = SnekAPI() diff --git a/tests/gunicorn_utils.py b/tests/gunicorn_utils.py index f2d9b6d..b417b1b 100644 --- a/tests/gunicorn_utils.py +++ b/tests/gunicorn_utils.py @@ -17,8 +17,8 @@ class _StandaloneApplication(Application): pass def load(self): - from snekbox.api.app import application - return application + from snekbox.api import SnekAPI + return SnekAPI() def load_config(self): for key, value in self.options.items(): -- cgit v1.2.3 From 416eabe3ac6224a17875ab2ae466666afc62ca4d Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 20:59:47 -0700 Subject: Specify wsgi_app in Gunicorn config --- Dockerfile | 2 +- config/gunicorn.conf.py | 1 + pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7331ab4..2382b8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ COPY config/ /snekbox/config/ FROM venv ENTRYPOINT ["gunicorn"] -CMD ["-c", "config/gunicorn.conf.py", "snekbox.api:SnekAPI"] +CMD ["-c", "config/gunicorn.conf.py"] COPY . /snekbox WORKDIR /snekbox diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 5812c07..8b40d35 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -3,3 +3,4 @@ bind = "0.0.0.0:8060" logger_class = "snekbox.utils.gunicorn.GunicornLogger" access_logformat = "%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss" access_logfile = "-" +wsgi_app = "snekbox.api:SnekAPI" diff --git a/pyproject.toml b/pyproject.toml index abcf04f..ac28b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ ] [project.optional-dependencies] -gunicorn = ["gunicorn>=20"] +gunicorn = ["gunicorn>=20.1"] # Lowest which supports wsgi_app in config. sentry = ["sentry-sdk[falcon]>=1.5.4"] [project.urls] -- cgit v1.2.3 From a53923652216b29bcdbf02eb66ecb36b83ec455a Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 21:11:02 -0700 Subject: Add __all__ to all modules --- config/gunicorn.conf.py | 2 +- snekbox/__init__.py | 8 ++++++-- snekbox/__main__.py | 2 +- snekbox/api/resources/eval.py | 2 ++ snekbox/nsjail.py | 2 ++ snekbox/utils/cgroup.py | 2 ++ snekbox/utils/swap.py | 2 ++ 7 files changed, 16 insertions(+), 4 deletions(-) diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 8b40d35..563f8ea 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -3,4 +3,4 @@ bind = "0.0.0.0:8060" logger_class = "snekbox.utils.gunicorn.GunicornLogger" access_logformat = "%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss" access_logfile = "-" -wsgi_app = "snekbox.api:SnekAPI" +wsgi_app = "snekbox:SnekAPI" diff --git a/snekbox/__init__.py b/snekbox/__init__.py index 1525ebd..ccb8b11 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,8 +1,12 @@ import os -from snekbox.utils.logging import init_logger, init_sentry - DEBUG = os.environ.get("DEBUG", False) +from snekbox.api import SnekAPI # noqa: E402 +from snekbox.nsjail import NsJail # noqa: E402 +from snekbox.utils.logging import init_logger, init_sentry # noqa: E402 + +__all__ = ("NsJail", "SnekAPI") + init_sentry() init_logger(DEBUG) diff --git a/snekbox/__main__.py b/snekbox/__main__.py index 704ec9d..7ac10e9 100644 --- a/snekbox/__main__.py +++ b/snekbox/__main__.py @@ -1,7 +1,7 @@ import argparse import sys -from snekbox.nsjail import NsJail +from snekbox import NsJail def parse_args() -> argparse.Namespace: diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py index 9560d0b..0a59f2e 100644 --- a/snekbox/api/resources/eval.py +++ b/snekbox/api/resources/eval.py @@ -5,6 +5,8 @@ from falcon.media.validators.jsonschema import validate from snekbox.nsjail import NsJail +__all__ = ("EvalResource",) + log = logging.getLogger(__name__) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index ac36551..1aca637 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -13,6 +13,8 @@ from google.protobuf import text_format from snekbox import DEBUG, utils from snekbox.config_pb2 import NsJailConfig +__all__ = ("NsJail",) + log = logging.getLogger(__name__) # [level][timestamp][PID]? function_signature:line_no? message diff --git a/snekbox/utils/cgroup.py b/snekbox/utils/cgroup.py index 3e12406..cd515ab 100644 --- a/snekbox/utils/cgroup.py +++ b/snekbox/utils/cgroup.py @@ -5,6 +5,8 @@ from snekbox.config_pb2 import NsJailConfig log = logging.getLogger(__name__) +__all__ = ("get_version", "init", "init_v1", "init_v2") + def get_version(config: NsJailConfig) -> int: """ diff --git a/snekbox/utils/swap.py b/snekbox/utils/swap.py index 3e0d0aa..6a919cb 100644 --- a/snekbox/utils/swap.py +++ b/snekbox/utils/swap.py @@ -4,6 +4,8 @@ from pathlib import Path from snekbox.config_pb2 import NsJailConfig +__all__ = ("controller_exists", "is_enabled", "should_ignore_limit") + log = logging.getLogger(__name__) -- cgit v1.2.3 From 67edafa0062582e4eec47c59d05a7b4b2429f435 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Mon, 30 May 2022 23:31:12 -0700 Subject: Automatically determine the package version Use the HEAD commit's date as the package's version. Append the number of commits made on the same date as HEAD to ensure multiple releases on the same date still have unique versions. --- pyproject.toml | 10 +++++++--- scripts/version.py | 34 ++++++++++++++++++++++++++++++++++ snekbox/__init__.py | 6 ++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 scripts/version.py diff --git a/pyproject.toml b/pyproject.toml index ac28b6e..c839da5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,9 @@ [build-system] -requires = ["setuptools>=61"] -build-backend = "setuptools.build_meta" +requires = ["setuptools>=61", "setuptools-git-versioning>=1.8"] +build-backend = "setuptools.build_meta:__legacy__" [project] name = "snekbox" -version = "1.0.0" description = "HTTP REST API for sanboxed execution of arbitrary Python code." readme = "README.md" license = {text = "MIT"} @@ -22,6 +21,7 @@ classifiers = [ "Topic :: Security", "Topic :: Software Development :: Interpreters", ] +dynamic = ["version"] requires-python = ">=3.10" dependencies = [ @@ -45,6 +45,10 @@ snekbox = "snekbox.__main__:main" [tool.setuptools] packages = ["snekbox"] +[tool.setuptools-git-versioning] +enabled = true +version_callback = "scripts.version:get_version" + [tool.coverage.report] exclude_lines = [ "pragma: no cover", diff --git a/scripts/version.py b/scripts/version.py new file mode 100644 index 0000000..9076b93 --- /dev/null +++ b/scripts/version.py @@ -0,0 +1,34 @@ +import datetime +import subprocess + +__all__ = ("get_version",) + + +def get_version() -> str: + """ + Return a version based on the HEAD commit's date. + + The format is 'year.month.day.commits' and is compliant with PEP 440. 'commits' is the amount of + commits made on the same date as HEAD, excluding HEAD. This ensures versions are unique if + multiple release occur on the same date. + """ + args = ["git", "show", "-s", "--format=%ct", "HEAD"] + stdout = subprocess.check_output(args, text=True) + timestamp = float(stdout.strip()) + date = datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) + + commits = count_commits_on_date(date) - 1 # Exclude HEAD. + + # Don't use strftime because it includes leading zeros, which are against PEP 440. + return f"{date.year}.{date.month}.{date.day}.{commits}" + + +def count_commits_on_date(dt: datetime.datetime) -> int: + """Return the amount of commits made on the given UTC aware datetime.""" + dt = dt.combine(dt - datetime.timedelta(days=1), dt.max.time(), dt.tzinfo) + + # git log uses the committer date for this, not the author date. + args = ["git", "log", "--oneline", "--after", str(dt.timestamp())] + stdout = subprocess.check_output(args, text=True) + + return stdout.strip().count("\n") diff --git a/snekbox/__init__.py b/snekbox/__init__.py index ccb8b11..a3a5f09 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,7 +1,13 @@ import os +from importlib import metadata DEBUG = os.environ.get("DEBUG", False) +try: + __version__ = metadata.version("snekbox") +except metadata.PackageNotFoundError: # pragma: no cover + __version__ = "0.0.0.0+unknown" + from snekbox.api import SnekAPI # noqa: E402 from snekbox.nsjail import NsJail # noqa: E402 from snekbox.utils.logging import init_logger, init_sentry # noqa: E402 -- cgit v1.2.3 From 9a60609b48376fda13593e515a3d237f2f50ff0c Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Tue, 31 May 2022 12:04:50 -0700 Subject: Fix package discovery for setuptools Subpackages weren't being included. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c839da5..fa61c1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,8 @@ tracker = "https://github.com/python-discord/snekbox/issues" [project.scripts] snekbox = "snekbox.__main__:main" -[tool.setuptools] -packages = ["snekbox"] +[tool.setuptools.packages.find] +include = ["snekbox*"] [tool.setuptools-git-versioning] enabled = true -- cgit v1.2.3 From 8d33cab2cb3baf179941e9692b682150d50bbbe5 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Tue, 31 May 2022 14:06:54 -0700 Subject: Docker: install package in image and use version to tag it --- .dockerignore | 8 ++++++-- .github/workflows/build.yaml | 24 ++++++++++++------------ .github/workflows/deploy.yaml | 7 +++---- .github/workflows/main.yaml | 4 ++-- .github/workflows/test.yaml | 6 +++--- Dockerfile | 19 ++++++++++--------- docker-compose.yml | 3 ++- scripts/version.py | 4 ++++ snekbox/__init__.py | 2 +- snekbox/utils/logging.py | 5 ++--- 10 files changed, 45 insertions(+), 37 deletions(-) diff --git a/.dockerignore b/.dockerignore index b2a96dd..6a360ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,9 +2,13 @@ * # Make exceptions for what's needed -!snekbox +!.git/ !config/ !requirements/ -!tests +!scripts/ +!snekbox/ +!tests/ !LICENSE +!NOTICE !pyproject.toml +!README.md diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 28e5b69..e5791c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,9 +4,9 @@ on: artifact: description: The name of the uploaded image aretfact. value: ${{ jobs.build.outputs.artifact }} - tag: - description: The tag used for the built image. - value: ${{ jobs.build.outputs.tag }} + version: + description: The package's version. + value: ${{ jobs.build.outputs.version }} jobs: build: @@ -14,21 +14,21 @@ jobs: runs-on: ubuntu-latest outputs: artifact: ${{ env.artifact }} - tag: ${{ steps.sha_tag.outputs.tag }} + version: ${{ steps.version.outputs.version }} env: artifact: image_artifact_snekbox-venv steps: - # Create a short SHA with which to tag built images. - - name: Create SHA Container Tag - id: sha_tag - run: | - tag=$(cut -c 1-7 <<< $GITHUB_SHA) - echo "::set-output name=tag::$tag" - - name: Checkout code uses: actions/checkout@v2 + - name: Get version + id: version + run: | + set -eu + version=$(python scripts/version.py) + echo "::set-output name=version::version" + # The current version (v2) of Docker's build-push action uses buildx, # which comes with BuildKit. It has cache features which can speed up # the builds. See https://github.com/docker/build-push-action @@ -83,7 +83,7 @@ jobs: ghcr.io/python-discord/snekbox-base:latest ghcr.io/python-discord/snekbox-venv:latest cache-to: ${{ steps.cache_config.outputs.cache_to }} - tags: ghcr.io/python-discord/snekbox-venv:${{ steps.sha_tag.outputs.tag }} + tags: ghcr.io/python-discord/snekbox-venv:${{ steps.version.outputs.version }} # Make the image available as an artifact so other jobs will be able to # download it. diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 3b12921..977c317 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -4,7 +4,7 @@ on: artifact: required: true type: string - tag: + version: required: true type: string secrets: @@ -55,8 +55,7 @@ jobs: cache-to: type=inline tags: | ghcr.io/python-discord/snekbox:latest - ghcr.io/python-discord/snekbox:${{ inputs.tag }} - build-args: git_sha=${{ github.sha }} + ghcr.io/python-discord/snekbox:${{ inputs.version }} # Deploy to Kubernetes. - name: Authenticate with Kubernetes @@ -69,7 +68,7 @@ jobs: uses: Azure/k8s-deploy@v1 with: manifests: deployment.yaml - images: 'ghcr.io/python-discord/snekbox:${{ inputs.tag }}' + images: 'ghcr.io/python-discord/snekbox:${{ inputs.version }}' kubectl-version: 'latest' # Push the base image to GHCR, with an inline cache manifest. diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f28ab61..b581ba3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,13 +16,13 @@ jobs: needs: build with: artifact: ${{ needs.build.outputs.artifact }} - tag: ${{ needs.build.outputs.tag }} + version: ${{ needs.build.outputs.version }} deploy: uses: ./.github/workflows/deploy.yaml if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} needs: [build, lint, test] with: artifact: ${{ needs.build.outputs.artifact }} - tag: ${{ needs.build.outputs.tag }} + version: ${{ needs.build.outputs.version }} secrets: KUBECONFIG: ${{ secrets.KUBECONFIG }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f7f3635..a9fcade 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,7 +4,7 @@ on: artifact: required: true type: string - tag: + version: required: true type: string @@ -38,7 +38,7 @@ jobs: - name: Run tests id: run_tests run: | - export IMAGE_SUFFIX='-venv:${{ inputs.tag }}' + export IMAGE_SUFFIX='-venv:${{ inputs.version }}' docker-compose run \ --rm -T -e COVERAGE_DATAFILE=.coverage.${{ matrix.os }} \ snekbox \ @@ -57,7 +57,7 @@ jobs: - name: Docker cleanup if: matrix.os == 'self-hosted' && always() run: | - export IMAGE_SUFFIX='-venv:${{ inputs.tag }}' + export IMAGE_SUFFIX='-venv:${{ inputs.version }}' docker-compose down --rmi all --remove-orphans -v -t 0 report: diff --git a/Dockerfile b/Dockerfile index 2382b8b..ef696e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM python:3.10-slim-buster as builder WORKDIR /nsjail @@ -31,6 +32,7 @@ ENV PATH=/root/.local/bin:$PATH \ RUN apt-get -y update \ && apt-get install -y \ gcc=4:8.3.* \ + git=1:2.20.* \ libnl-route-3-200=3.4.* \ libprotobuf17=3.6.* \ && rm -rf /var/lib/apt/lists/* @@ -60,18 +62,17 @@ RUN if [ -n "${DEV}" ]; \ fi # At the end to avoid re-installing dependencies when only a config changes. -# It's in the venv image because the final image is not used during development. COPY config/ /snekbox/config/ -# ------------------------------------------------------------------------------ -FROM venv - ENTRYPOINT ["gunicorn"] CMD ["-c", "config/gunicorn.conf.py"] -COPY . /snekbox -WORKDIR /snekbox +# ------------------------------------------------------------------------------ +FROM venv -# At the end to prevent it from invalidating the layer cache. -ARG git_sha="development" -ENV GIT_SHA=$git_sha +# Use a separate directory to avoid importing the source over the instaled pkg. +# The venv already installed dependencies, so nothing besides snekbox itself +# will be installed. Note requirements.pip cannot be used as a constraint file +# because it contains extras, which pip disallows. +RUN --mount=source=.,target=/snekbox_src,rw \ + pip install /snekbox_src[gunicorn,sentry] \ diff --git a/docker-compose.yml b/docker-compose.yml index 3854825..aa1a0f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: snekbox_dev hostname: snekbox_dev privileged: true - image: ghcr.io/python-discord/snekbox${IMAGE_SUFFIX:-:dev} + image: ghcr.io/python-discord/snekbox${IMAGE_SUFFIX:--venv:dev} ports: - 8060:8060 init: true @@ -17,6 +17,7 @@ services: build: context: . dockerfile: Dockerfile + target: venv args: DEV: 1 cache_from: diff --git a/scripts/version.py b/scripts/version.py index 9076b93..bf8d509 100644 --- a/scripts/version.py +++ b/scripts/version.py @@ -32,3 +32,7 @@ def count_commits_on_date(dt: datetime.datetime) -> int: stdout = subprocess.check_output(args, text=True) return stdout.strip().count("\n") + + +if __name__ == "__main__": + print(get_version()) diff --git a/snekbox/__init__.py b/snekbox/__init__.py index a3a5f09..657c032 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -14,5 +14,5 @@ from snekbox.utils.logging import init_logger, init_sentry # noqa: E402 __all__ = ("NsJail", "SnekAPI") -init_sentry() +init_sentry(__version__) init_logger(DEBUG) diff --git a/snekbox/utils/logging.py b/snekbox/utils/logging.py index 9a713f8..c15e3f1 100644 --- a/snekbox/utils/logging.py +++ b/snekbox/utils/logging.py @@ -19,7 +19,7 @@ def init_logger(debug: bool) -> None: log.addHandler(handler) -def init_sentry() -> None: +def init_sentry(version: str) -> None: """Initialise the Sentry SDK if it's installed.""" try: import sentry_sdk @@ -27,10 +27,9 @@ def init_sentry() -> None: except ImportError: return - git_sha = os.environ.get("GIT_SHA", "development") sentry_sdk.init( dsn=os.environ.get("SNEKBOX_SENTRY_DSN", ""), integrations=[FalconIntegration()], send_default_pii=True, - release=f"snekbox@{git_sha}" + release=f"snekbox@{version}" ) -- cgit v1.2.3 From e13329e0311516085aaea3ef6b73953800b5a87d Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Tue, 31 May 2022 14:08:21 -0700 Subject: CI: tag all images with version --- .github/workflows/deploy.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 977c317..9113188 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -81,7 +81,9 @@ jobs: push: true cache-from: ghcr.io/python-discord/snekbox-base:latest cache-to: type=inline - tags: ghcr.io/python-discord/snekbox-base:latest + tags: | + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-base:${{ inputs.version }} # Push the venv image to GHCR, with an inline cache manifest. - name: Push venv image @@ -95,4 +97,6 @@ jobs: ghcr.io/python-discord/snekbox-base:latest ghcr.io/python-discord/snekbox-venv:latest cache-to: type=inline - tags: ghcr.io/python-discord/snekbox-venv:latest + tags: | + ghcr.io/python-discord/snekbox-venv:latest + ghcr.io/python-discord/snekbox-venv:${{ inputs.version }} -- cgit v1.2.3 From 4c58f8bd5db220223c45955dfd01bcf496e46cdd Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Tue, 31 May 2022 14:34:18 -0700 Subject: CI: fix container entrypoint when running tests --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a9fcade..30e6ba3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,8 +41,9 @@ jobs: export IMAGE_SUFFIX='-venv:${{ inputs.version }}' docker-compose run \ --rm -T -e COVERAGE_DATAFILE=.coverage.${{ matrix.os }} \ + --entrypoint coverage \ snekbox \ - coverage run -m unittest + run -m unittest # Upload it so the coverage from all matrix jobs can be combined later. - name: Upload coverage data -- cgit v1.2.3 From c2961a3b5d061904c7846c32af0d98cdfd3d9692 Mon Sep 17 00:00:00 2001 From: MarkKoz <1515135+MarkKoz@users.noreply.github.com> Date: Tue, 31 May 2022 14:49:05 -0700 Subject: Fix typo in project description --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fa61c1a..08aeb8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "snekbox" -description = "HTTP REST API for sanboxed execution of arbitrary Python code." +description = "HTTP REST API for sandboxed execution of arbitrary Python code." readme = "README.md" license = {text = "MIT"} authors = [{name = "Python Discord", email = "info@pythondiscord.com"}] -- cgit v1.2.3 From 6ba918290ba6bc9bac4265b8e6bc3b83eed92e36 Mon Sep 17 00:00:00 2001 From: Mark <1515135+MarkKoz@users.noreply.github.com> Date: Wed, 1 Jun 2022 08:01:33 -0700 Subject: Fix typo in Dockerfile comment Co-authored-by: Matteo Bertucci --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef696e0..e6110d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ CMD ["-c", "config/gunicorn.conf.py"] # ------------------------------------------------------------------------------ FROM venv -# Use a separate directory to avoid importing the source over the instaled pkg. +# Use a separate directory to avoid importing the source over the installed pkg. # The venv already installed dependencies, so nothing besides snekbox itself # will be installed. Note requirements.pip cannot be used as a constraint file # because it contains extras, which pip disallows. -- cgit v1.2.3