From 528ac7d750aeeb6daba1826b5f1d6c246a93b0a4 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 Jan 2021 22:14:29 -0800 Subject: Docker: move env var definitions to the base stage Just a refactor. --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eefb795..a115485 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,11 @@ WORKDIR /nsjail RUN make FROM python:3.9-slim-buster as base -ENV PIP_NO_CACHE_DIR=false +ENV PIP_NO_CACHE_DIR=false \ + PIPENV_DONT_USE_PYENV=1 \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_NOSPIN=1 \ + PYTHONUSERBASE=/snekbox/user_base RUN apt-get -y update \ && apt-get install -y \ -- cgit v1.2.3 From cfc1f73daaaaaa9564e34f3e4e855acb19ad9e21 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 Jan 2021 22:19:13 -0800 Subject: Docker: add comments to separate stages --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index a115485..a6ebe2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN git clone \ WORKDIR /nsjail RUN make +# ------------------------------------------------------------------------------ FROM python:3.9-slim-buster as base ENV PIP_NO_CACHE_DIR=false \ PIPENV_DONT_USE_PYENV=1 \ @@ -37,6 +38,7 @@ RUN pip install pipenv==2020.11.4 COPY --from=builder /nsjail/nsjail /usr/sbin/ RUN chmod +x /usr/sbin/nsjail +# ------------------------------------------------------------------------------ FROM base as venv ARG DEV @@ -60,6 +62,7 @@ RUN if [ -n "${DEV}" ]; \ # It's in the venv image because the final image is not used during development. COPY config/ /snekbox/config +# ------------------------------------------------------------------------------ FROM venv ENTRYPOINT ["gunicorn"] -- cgit v1.2.3 From c2a3f316c9e4ae95807d3a464b9cc99ad2de0107 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 Jan 2021 22:34:50 -0800 Subject: Docker: simplify pipenv install command --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a6ebe2d..62d8bcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,12 +51,7 @@ ENV PIP_NO_CACHE_DIR=false \ COPY Pipfile Pipfile.lock /snekbox/ WORKDIR /snekbox -RUN if [ -n "${DEV}" ]; \ - then \ - pipenv install --deploy --system --dev; \ - else \ - pipenv install --deploy --system; \ - fi +RUN pipenv install --deploy --system ${DEV:+--dev} # At the end to avoid re-installing dependencies when only a config changes. # It's in the venv image because the final image is not used during development. -- cgit v1.2.3 From e73e989198b097989fc069f13ca166fb8eedbfb8 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 12 Jan 2021 10:31:41 -0800 Subject: Fix caching of exposed packages The problem with --ignore-installed is that it also ignores packages in the target site, therefore re-installing all packages and making caching futile. Install snekbox's dependencies to the default user site. During installation of the exposed packages, switch the user site to a custom one to ensure pip will not skip packages due to seeing them as already installed as one of snekbox's dependencies. If pip made the --root option ignore packages outside the root, then using --root would be the best solution. There is a 5+ year old issue open about that. --- Dockerfile | 17 ++++++++--------- README.md | 10 ++++------ deployment.yaml | 3 ++- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 62d8bcb..87ece7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,11 +21,15 @@ RUN make # ------------------------------------------------------------------------------ FROM python:3.9-slim-buster as base -ENV PIP_NO_CACHE_DIR=false \ + +# Everything will be a user install to allow snekbox's dependencies to be kept +# separate from the packages exposed during eval. +ENV PATH=/root/.local/bin:$PATH \ + PIP_NO_CACHE_DIR=false \ + PIP_USER=1 \ PIPENV_DONT_USE_PYENV=1 \ PIPENV_HIDE_EMOJIS=1 \ - PIPENV_NOSPIN=1 \ - PYTHONUSERBASE=/snekbox/user_base + PIPENV_NOSPIN=1 RUN apt-get -y update \ && apt-get install -y \ @@ -42,15 +46,10 @@ RUN chmod +x /usr/sbin/nsjail FROM base as venv ARG DEV -ENV PIP_NO_CACHE_DIR=false \ - PIPENV_DONT_USE_PYENV=1 \ - PIPENV_HIDE_EMOJIS=1 \ - PIPENV_NOSPIN=1 \ - PYTHONUSERBASE=/snekbox/user_base - COPY Pipfile Pipfile.lock /snekbox/ WORKDIR /snekbox +# Install to the default user site since PIP_USER is set. RUN pipenv install --deploy --system ${DEV:+--dev} # At the end to avoid re-installing dependencies when only a config changes. diff --git a/README.md b/README.md index 453eb4c..f59cf8b 100644 --- a/README.md +++ b/README.md @@ -50,19 +50,17 @@ By default, the Python interpreter has no access to any packages besides the standard library. Even snekbox's own dependencies like Falcon and Gunicorn are not exposed. -To expose third-party Python packages during evaluation, install them to the user site: +To expose third-party Python packages during evaluation, install them to a custom user site: ```sh -docker exec snekbox /bin/sh -c 'pip install --ignore-installed --user numpy' +docker exec snekbox /bin/sh -c 'PYTHONUSERBASE=/snekbox/user_base pip install numpy' ``` In the above command, `snekbox` is the name of the running container. The name may be different and can be checked with `docker ps`. -It's important to use `--user` to install them to the user site, whose base is located at `/snekbox/user_base` within the Docker container. To persist the installed packages, a volume for the directory can be created with Docker. For an example, see [`docker-compose.yml`]. +The packages will be installed to the user site within `/snekbox/user_base`. To persist the installed packages, a volume for the directory can be created with Docker. For an example, see [`docker-compose.yml`]. -`--ignore-installed` is only necessary if installing a package that happens to -be a dependency of snekbox. Normally, pip would reject the installation because -it doesn't make a distinction here between the global and user sites. +If `pip`, `setuptools`, or `wheel` are dependencies or need to be exposed, then use the `--ignore-installed` option with pip. However, note that this will also re-install packages present in the custom user site, effectively making caching it futile. Current limitations of pip don't allow it to ignore packages extant outside the installation destination. ## Development Environment diff --git a/deployment.yaml b/deployment.yaml index 988394f..0b294d2 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -30,7 +30,8 @@ spec: - "/bin/sh" - "-c" - >- - pip install --user --ignore-installed + PYTHONUSERBASE=/snekbox/user_base + pip install --user arrow~=0.17 attrs~=20.3 beautifulsoup4~=4.9 -- cgit v1.2.3 From 9f1d1c670eb0cb1b09657bcb5ed28b3db6338457 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 12 Jan 2021 10:35:43 -0800 Subject: Docker: update pipenv --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 87ece7d..2aee11b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN apt-get -y update \ libnl-route-3-200=3.4.* \ libprotobuf17=3.6.* \ && rm -rf /var/lib/apt/lists/* -RUN pip install pipenv==2020.11.4 +RUN pip install pipenv==2020.11.15 COPY --from=builder /nsjail/nsjail /usr/sbin/ RUN chmod +x /usr/sbin/nsjail -- cgit v1.2.3 From 247eab38e4142918da9aa25555f4f72382faeb2f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 12 Jan 2021 11:57:10 -0800 Subject: CI: don't do a user install for pre-commit --- .github/workflows/lint-test-build-push.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-test-build-push.yaml b/.github/workflows/lint-test-build-push.yaml index 5aa2180..700fba9 100644 --- a/.github/workflows/lint-test-build-push.yaml +++ b/.github/workflows/lint-test-build-push.yaml @@ -108,11 +108,12 @@ jobs: docker exec snekbox_test /bin/bash -c 'apt-get -y update && apt-get install -y git=1:2.20.*' + # pre-commit's venv doesn't work with user installs. # Skip the flake8 hook because the following step will run it. - name: Run pre-commit hooks run: >- docker exec snekbox_test /bin/bash -c - 'SKIP=flake8 pre-commit run --all-files' + 'PIP_USER=0 SKIP=flake8 pre-commit run --all-files' # This runs `flake8` in the container and asks `flake8` to output # linting errors in the format of the command for registering workflow -- cgit v1.2.3