diff options
| -rw-r--r-- | .dockerignore | 2 | ||||
| -rw-r--r-- | .github/CONTRIBUTING.md | 8 | ||||
| -rw-r--r-- | Dockerfile | 82 | ||||
| -rw-r--r-- | LICENSE-THIRD-PARTY (renamed from NOTICE) | 39 | ||||
| -rw-r--r-- | README.md | 12 | ||||
| -rw-r--r-- | config/snekbox.cfg | 33 | ||||
| -rw-r--r-- | deployment.yaml | 6 | ||||
| -rwxr-xr-x | scripts/build_python.sh | 22 | ||||
| -rw-r--r-- | snekbox/nsjail.py | 4 | ||||
| -rw-r--r-- | tests/test_nsjail.py | 4 |
10 files changed, 135 insertions, 77 deletions
diff --git a/.dockerignore b/.dockerignore index 6a360ff..ab815cd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,6 @@ !snekbox/ !tests/ !LICENSE -!NOTICE +!LICENSE-THIRD-PARTY !pyproject.toml !README.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d0a6921..1124b8e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -62,6 +62,14 @@ Updating NsJail mainly involves two steps: Other things to look out for are breaking changes to NsJail's config format, its command-line interface, or its logging format. Additionally, dependencies may have to be adjusted in the Dockerfile to get a new version to build or run. +## Adding and Updating Python Interpreters + +Python interpreters are built using pyenv via the `scripts/build_python.sh` helper script. This script accepts a pyenv version specifier (`pyenv install --list`) and builds the interpreter in a version-specific directory under `/lang/python`. In the image, each minor version of a Python interpreter should have its own build stage and the resulting `/lang/python` directory can be copied from that stage into the `base` stage. + +When updating a patch version (e.g. 3.11.3 to 3.11.4), edit the existing build stage in the image for the minor version (3.11); do not add a new build stage. To have access to a new version, pyenv likely needs to be updated. To do so, change the tag in the `git clone` command in the image, but only for the build stage that needs access to the new version. Updating pyenv for all build stages will just cause unnecessary build cache invalidations. + +To change the default interpreter used by NsJail, update the target of the `/lang/python/default` symlink created in the `base` stage. + [readme]: ../README.md [Dockerfile]: ../Dockerfile [Compose v2]: https://docs.docker.com/compose/compose-v2/ @@ -1,52 +1,73 @@ -# syntax=docker/dockerfile:1 -FROM python:3.11-slim-buster as builder +# syntax=docker/dockerfile:1.4 +FROM buildpack-deps:buster as builder-nsjail WORKDIR /nsjail RUN apt-get -y update \ - && apt-get install -y \ - bison=2:3.3.* \ - flex=2.6.* \ - g++=4:8.3.* \ - gcc=4:8.3.* \ - git=1:2.20.* \ - libprotobuf-dev=3.6.* \ - libnl-route-3-dev=3.4.* \ - make=4.2.* \ - pkg-config=0.29-6 \ - protobuf-compiler=3.6.* + && apt-get install -y --no-install-recommends \ + bison\ + flex \ + libprotobuf-dev\ + libnl-route-3-dev \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 RUN make # ------------------------------------------------------------------------------ +FROM buildpack-deps:buster as builder-py-base + +ENV PYENV_ROOT=/pyenv \ + PYTHON_CONFIGURE_OPTS='--disable-test-modules --enable-optimizations \ + --with-lto --with-system-expat --without-ensurepip' + +RUN apt-get -y update \ + && apt-get install -y --no-install-recommends \ + libxmlsec1-dev \ + tk-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --link scripts/build_python.sh / + +# ------------------------------------------------------------------------------ +FROM builder-py-base as builder-py-3_11 +RUN git clone -b v2.3.24 --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT \ + && /build_python.sh 3.11.4 + +# ------------------------------------------------------------------------------ +FROM builder-py-base as builder-py-3_12 +RUN git clone -b v2.3.24 --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT \ + && /build_python.sh 3.12.0rc1 + +# ------------------------------------------------------------------------------ FROM python:3.11-slim-buster as base -# Everything will be a user install to allow snekbox's dependencies to be kept -# separate from the packages exposed during eval. -ENV PATH=/root/.local/bin:$PATH \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_NO_CACHE_DIR=false \ - PIP_USER=1 +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=false RUN apt-get -y update \ - && apt-get install -y \ - gcc=4:8.3.* \ - git=1:2.20.* \ - libnl-route-3-200=3.4.* \ - libprotobuf17=3.6.* \ + && apt-get install -y --no-install-recommends \ + gcc \ + git \ + libnl-route-3-200 \ + libprotobuf17 \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /nsjail/nsjail /usr/sbin/ -RUN chmod +x /usr/sbin/nsjail +COPY --link --from=builder-nsjail /nsjail/nsjail /usr/sbin/ +COPY --link --from=builder-py-3_11 /lang/ /lang/ +COPY --link --from=builder-py-3_12 /lang/ /lang/ + +RUN chmod +x /usr/sbin/nsjail \ + && ln -s /lang/python/3.11/ /lang/python/default # ------------------------------------------------------------------------------ FROM base as venv -COPY requirements/ /snekbox/requirements/ +COPY --link requirements/ /snekbox/requirements/ WORKDIR /snekbox -# pip installs to the default user site since PIP_USER is set. RUN pip install -U -r requirements/requirements.pip # This must come after the first pip command! From the docs: @@ -58,11 +79,12 @@ ARG DEV RUN if [ -n "${DEV}" ]; \ then \ pip install -U -r requirements/coverage.pip \ - && PYTHONUSERBASE=/snekbox/user_base pip install numpy~=1.19; \ + && export PYTHONUSERBASE=/snekbox/user_base \ + && /lang/python/default/bin/python -m pip install --user numpy~=1.19; \ fi # At the end to avoid re-installing dependencies when only a config changes. -COPY config/ /snekbox/config/ +COPY --link config/ /snekbox/config/ ENTRYPOINT ["gunicorn"] CMD ["-c", "config/gunicorn.conf.py"] diff --git a/NOTICE b/LICENSE-THIRD-PARTY index b6e5fbc..684f2df 100644 --- a/NOTICE +++ b/LICENSE-THIRD-PARTY @@ -1,9 +1,36 @@ -The Python code at snekbox/config_pb2.py was generated from config.proto in nsjail -Copyright 2014 Google Inc. All Rights Reserved. -Copyright 2016 Sergiusz Bazanski. All Rights Reserved. - -------------------------------------------------------------------------------- - +-------------------------------------------------------------------------------- + MIT License +Applies to: + - Copyright (c) 2014 Docker, Inc. + - scripts/build_python.sh: find command for de-bloating Python install +-------------------------------------------------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + Apache License, Version 2.0 +Applies to: + - Copyright 2014 Google Inc. All Rights Reserved. + Copyright 2016 Sergiusz Bazanski. All Rights Reserved. + - snekbox/config_pb2.py: generated from config.proto in nsjail +-------------------------------------------------------------------------------- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -7,8 +7,7 @@ Python sandbox runners for executing code in isolation aka snekbox. -Supports a memory [virtual read/write file system](#virtual-file-system) within the sandbox, -allowing text or binary files to be sent and returned. +Supports a memory [virtual read/write file system](#virtual-file-system) within the sandbox, allowing text or binary files to be sent and returned. A client sends Python code to a snekbox, the snekbox executes the code, and finally the results of the execution are returned to the client. @@ -100,22 +99,19 @@ Name | Description ## Third-party Packages -By default, the Python interpreter has no access to any packages besides the -standard library. Even snekbox's own dependencies like Falcon and Gunicorn are -not exposed. +By default, the Python interpreter has no access to any packages besides the standard library. Even snekbox's own dependencies like Falcon and Gunicorn are not exposed. To expose third-party Python packages during evaluation, install them to a custom user site: ```sh -docker exec snekbox /bin/sh -c 'PYTHONUSERBASE=/snekbox/user_base pip install numpy' +docker exec snekbox /bin/sh -c \ + 'PYTHONUSERBASE=/snekbox/user_base /lang/python/default/bin/python -m pip install --user numpy' ``` In the above command, `snekbox` is the name of the running container. The name may be different and can be checked with `docker ps`. The packages will be installed to the user site within `/snekbox/user_base`. To persist the installed packages, a volume for the directory can be created with Docker. For an example, see [`docker-compose.yml`]. -If `pip`, `setuptools`, or `wheel` are dependencies or need to be exposed, then use the `--ignore-installed` option with pip. However, note that this will also re-install packages present in the custom user site, effectively making caching it futile. Current limitations of pip don't allow it to ignore packages extant outside the installation destination. - ## Development Environment See [CONTRIBUTING.md](.github/CONTRIBUTING.md). diff --git a/config/snekbox.cfg b/config/snekbox.cfg index 5dd63da..4e146ec 100644 --- a/config/snekbox.cfg +++ b/config/snekbox.cfg @@ -14,8 +14,10 @@ envar: "OPENBLAS_NUM_THREADS=5" envar: "MKL_NUM_THREADS=5" envar: "VECLIB_MAXIMUM_THREADS=5" envar: "NUMEXPR_NUM_THREADS=5" -envar: "PYTHONPATH=/snekbox/user_base/lib/python3.11/site-packages" +envar: "PYTHONDONTWRITEBYTECODE=true" envar: "PYTHONIOENCODING=utf-8:strict" +envar: "PYTHONUNBUFFERED=true" +envar: "PYTHONUSERBASE=/snekbox/user_base" envar: "HOME=home" keep_caps: false @@ -79,29 +81,8 @@ mount { } mount { - src: "/usr/local/lib" - dst: "/usr/local/lib" - is_bind: true - rw: false -} - -mount { - src: "/usr/local/bin/python" - dst: "/usr/local/bin/python" - is_bind: true - rw: false -} - -mount { - src: "/usr/local/bin/python3" - dst: "/usr/local/bin/python3" - is_bind: true - rw: false -} - -mount { - src: "/usr/local/bin/python3.11" - dst: "/usr/local/bin/python3.11" + src: "/lang" + dst: "/lang" is_bind: true rw: false } @@ -116,6 +97,6 @@ cgroup_pids_mount: "/sys/fs/cgroup/pids" iface_no_lo: true exec_bin { - path: "/usr/local/bin/python" - arg: "-BSqu" + path: "/lang/python/default/bin/python" + arg: "" } diff --git a/deployment.yaml b/deployment.yaml index b0856f3..a6e7a31 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -30,8 +30,9 @@ spec: - "/bin/sh" - "-c" - >- - PYTHONUSERBASE=/snekbox/user_base - pip install --user --upgrade + find /lang/python -mindepth 1 -maxdepth 1 -type d -exec + sh -c 'PYTHONUSERBASE=/snekbox/user_base && + {}/bin/python -m pip install --user -U anyio[trio]~=3.6 arrow~=1.2 attrs~=22.2 @@ -55,6 +56,7 @@ spec: typing-extensions~=4.4 tzdata~=2022.7 yarl~=1.8 + ' \; volumes: - name: snekbox-user-base-volume hostPath: diff --git a/scripts/build_python.sh b/scripts/build_python.sh new file mode 100755 index 0000000..da937c2 --- /dev/null +++ b/scripts/build_python.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euxo pipefail +shopt -s inherit_errexit + +py_version="${1}" + +# Install Python interpreter under e.g. /lang/python/3.11/ (no patch version). +"${PYENV_ROOT}/plugins/python-build/bin/python-build" \ + "${py_version}" \ + "/lang/python/${py_version%.*}" +"/lang/python/${py_version%.*}/bin/python" -m pip install -U pip + +# Clean up some unnecessary files to reduce image size bloat. +find /lang/python/ -depth \ +\( \ + \( -type d -a \( \ + -name test -o -name tests -o -name idle_test \ + \) \) \ + -o \( -type f -a \( \ + -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \ + \) \) \ +\) -exec rm -rf '{}' + diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index f64830a..1de7b1e 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -221,9 +221,9 @@ class NsJail: *nsjail_args, "--", self.config.exec_bin.path, - *self.config.exec_bin.arg, - # Filter out empty strings at start of py_args + # Filter out empty strings at start of Python args # (causes issues with python cli) + *iter_lstrip(self.config.exec_bin.arg), *iter_lstrip(py_args), ] diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py index 5b06534..e422de5 100644 --- a/tests/test_nsjail.py +++ b/tests/test_nsjail.py @@ -79,7 +79,7 @@ class NsJailTests(unittest.TestCase): for _ in range({max_pids}): print(subprocess.Popen( [ - '/usr/local/bin/python3', + '/lang/python/default/bin/python', '-c', 'import time; time.sleep(1)' ], @@ -486,7 +486,7 @@ class NsJailTests(unittest.TestCase): for args, expected in cases: with self.subTest(args=args): result = self.nsjail.python3(py_args=args) - idx = result.args.index("-BSqu") + idx = result.args.index(self.nsjail.config.exec_bin.path) self.assertEqual(result.args[idx + 1 :], expected) self.assertEqual(result.returncode, 0) |