Merge pull request #31 from python-discord/testing

NsJail Tests

12 files changed, 364 insertions, 191 deletions

diff --git a/.coveragerc b/.coveragerc
index d1877d4..8490bab 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,6 +1,9 @@
 [run]
 branch = True
-omit =
-    .venv/*,
-    snekbox/api/app.py,
-    tests/*
+include = snekbox/*
+omit = snekbox/api/app.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    if DEBUG
diff --git a/.dockerignore b/.dockerignore
index 2a5ccec..afc786a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,7 +2,9 @@
 *
 
 # Make exceptions for what's needed
+!docker/.profile
 !snekbox
+!tests
 !Pipfile
 !Pipfile.lock
 !LICENSE
diff --git a/Pipfile b/Pipfile
index 986116d..1aa46a8 100644
--- a/Pipfile
+++ b/Pipfile
@@ -9,9 +9,7 @@ gunicorn = "*"
 jsonschema = "*"
 
 [dev-packages]
-pytest = "*"
-pytest-cov = "*"
-pytest-dependency = "*"
+coverage = "*"
 pre-commit = "*"
 flake8 = "*"
 flake8-docstrings = "*"
@@ -22,6 +20,7 @@ flake8-todo = "*"
 flake8-string-format = "*"
 flake8-formatter-junit-xml = "*"
 flake8-quotes = "*"
+unittest-xml-reporting = "*"
 
 [requires]
 python_version = "3.7"
@@ -29,17 +28,18 @@ python_version = "3.7"
 [scripts]
 lint = "flake8"
 precommit = "pre-commit install"
-test = "pytest tests --cov . --cov-report term-missing -v"
-report = "pytest tests --cov . --cov-report=html"
+test = "scripts/dev.sh -c 'pipenv run coverage run -m unittest'"
+report = "coverage html"
 snekbox = """
-    gunicorn
-        -w 2
-        -b 0.0.0.0:8060
-        --logger-class snekbox.GunicornLogger
-        --access-logformat '%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss'
-        --access-logfile -
+    gunicorn \
+        -w 2 \
+        -b 0.0.0.0:8060 \
+        --logger-class snekbox.GunicornLogger \
+        --access-logformat '%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss' \
+        --access-logfile - \
         snekbox.api.app
 """
+devsh = "scripts/dev.sh"
 buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ."
 pushbox = "docker push pythondiscord/snekbox:latest"
 buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ."
diff --git a/Pipfile.lock b/Pipfile.lock
index c09c916..4f6bef8 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "af53793d2c00001698021096041ed23e5de0a5553f974822e9ad7e58f70de4a9"
+            "sha256": "421cdae80970f990af48506e38464e4e2b99e20291070943027b86bd7ca29c5b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -81,13 +81,6 @@
             ],
             "version": "==1.3.0"
         },
-        "atomicwrites": {
-            "hashes": [
-                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
-                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
-            ],
-            "version": "==1.3.0"
-        },
         "attrs": {
             "hashes": [
                 "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
@@ -136,6 +129,7 @@
                 "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
                 "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
             ],
+            "index": "pypi",
             "version": "==4.5.3"
         },
         "entrypoints": {
@@ -224,17 +218,17 @@
         },
         "identify": {
             "hashes": [
-                "sha256:432c548d6138cb57a3d8f62f079a025a29b8ae34a50dd3b496bbf661818f2bc0",
-                "sha256:d4401d60bf1938aa3074a352a5cc9044107edf11a6fedd3a1db172c141619b81"
+                "sha256:0a11379b46d06529795442742a043dc2fa14cd8c995ae81d1febbc5f1c014c87",
+                "sha256:43a5d24ffdb07bc7e21faf68b08e9f526a1f41f0056073f480291539ef961dfd"
             ],
-            "version": "==1.4.3"
+            "version": "==1.4.5"
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:a9f185022cfa69e9ca5f7eabfd5a58b689894cb78a11e3c8c89398a8ccbb8e7f",
-                "sha256:df1403cd3aebeb2b1dcd3515ca062eecb5bd3ea7611f18cba81130c68707e879"
+                "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
+                "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
             ],
-            "version": "==0.17"
+            "version": "==0.18"
         },
         "junit-xml": {
             "hashes": [
@@ -249,48 +243,19 @@
             ],
             "version": "==0.6.1"
         },
-        "more-itertools": {
-            "hashes": [
-                "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7",
-                "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"
-            ],
-            "markers": "python_version > '2.7'",
-            "version": "==7.0.0"
-        },
         "nodeenv": {
             "hashes": [
                 "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a"
             ],
             "version": "==1.3.3"
         },
-        "packaging": {
-            "hashes": [
-                "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af",
-                "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"
-            ],
-            "version": "==19.0"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc",
-                "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"
-            ],
-            "version": "==0.12.0"
-        },
         "pre-commit": {
             "hashes": [
-                "sha256:6ca409d1f22d444af427fb023a33ca8b69625d508a50e1b7eaabd59247c93043",
-                "sha256:94dd519597f5bff06a4b0df194a79c524b78f4b1534c1ce63241a9d4fb23b926"
+                "sha256:92e406d556190503630fd801958379861c94884693a032ba66629d0351fdccd4",
+                "sha256:cccc39051bc2457b0c0f7152a411f8e05e3ba2fe1a5613e4ee0833c1c1985ce3"
             ],
             "index": "pypi",
-            "version": "==1.16.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
-                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
-            ],
-            "version": "==1.8.0"
+            "version": "==1.17.0"
         },
         "pycodestyle": {
             "hashes": [
@@ -314,51 +279,21 @@
             ],
             "version": "==2.1.1"
         },
-        "pyparsing": {
-            "hashes": [
-                "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
-                "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
-            ],
-            "version": "==2.4.0"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:6032845e68a17a96e8da3088037f899b56357769a724122056265ca2ea1890ee",
-                "sha256:bea27a646a3d74cbbcf8d3d4a06b2dfc336baf3dc2cc85cf70ad0157e73e8322"
-            ],
-            "index": "pypi",
-            "version": "==4.6.2"
-        },
-        "pytest-cov": {
-            "hashes": [
-                "sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6",
-                "sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a"
-            ],
-            "index": "pypi",
-            "version": "==2.7.1"
-        },
-        "pytest-dependency": {
-            "hashes": [
-                "sha256:bda0ef48e6a44c091399b12ab4a7e580d2dd8294c222b301f88d7d57f47ba142"
-            ],
-            "index": "pypi",
-            "version": "==0.4.0"
-        },
         "pyyaml": {
             "hashes": [
-                "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c",
-                "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95",
-                "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2",
-                "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4",
-                "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad",
-                "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba",
-                "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1",
-                "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e",
-                "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673",
-                "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13",
-                "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19"
+                "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
+                "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
+                "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
+                "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
+                "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
+                "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
+                "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
+                "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
+                "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
+                "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
+                "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
             ],
-            "version": "==5.1"
+            "version": "==5.1.1"
         },
         "six": {
             "hashes": [
@@ -381,19 +316,20 @@
             ],
             "version": "==0.10.0"
         },
-        "virtualenv": {
+        "unittest-xml-reporting": {
             "hashes": [
-                "sha256:99acaf1e35c7ccf9763db9ba2accbca2f4254d61d1912c5ee364f9cc4a8942a0",
-                "sha256:fe51cdbf04e5d8152af06c075404745a7419de27495a83f0d72518ad50be3ce8"
+                "sha256:140982e4b58e4052d9ecb775525b246a96bfc1fc26097806e05ea06e9166dd6c",
+                "sha256:d1fbc7a1b6c6680ccfe75b5e9701e5431c646970de049e687b4bb35ba4325d72"
             ],
-            "version": "==16.6.0"
+            "index": "pypi",
+            "version": "==2.5.1"
         },
-        "wcwidth": {
+        "virtualenv": {
             "hashes": [
-                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
-                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
+                "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a",
+                "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783"
             ],
-            "version": "==0.1.7"
+            "version": "==16.6.1"
         },
         "zipp": {
             "hashes": [
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index bd916a4..98d64bf 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -2,32 +2,73 @@
 
 jobs:
 - job: test
-  displayName: 'Lint'
+  displayName: 'Lint & Test'
 
   pool:
     vmImage: 'Ubuntu-16.04'
 
   steps:
-  - task: UsePythonVersion@0
-    displayName: 'Set Python version'
-    inputs:
-      versionSpec: '3.7.x'
-      addToPath: true
+  - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile .
+    displayName: 'Build Base Image'
+
+  - script: |
+      docker build -t pythondiscord/snekbox-venv:dev -f docker/venv.Dockerfile --build-arg DEV=1 .
+    displayName: 'Build Development Image'
+
+  - script: |
+      docker run \
+        -td \
+        --name snekbox_test \
+        --privileged \
+        --network host \
+        -h pdsnk-dev \
+        -e PYTHONDONTWRITEBYTECODE=1 \
+        -e PIPENV_PIPFILE="/snekbox/Pipfile" \
+        -e ENV="${PWD}/scripts/.profile" \
+        -v "${PWD}":"${PWD}" \
+        -w "${PWD}"\
+        --entrypoint /bin/ash \
+        pythondiscord/snekbox-venv:dev
+    displayName: 'Start Container'
+
+  - script: |
+      docker exec snekbox_test /bin/ash -c \
+        'pipenv run lint --format junit-xml --output-file test-lint.xml'
+    displayName: 'Run Linter'
 
-  - script: pip3 install pipenv
-    displayName: 'Install pipenv'
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    displayName: 'Publish Lint Results'
+    inputs:
+      testResultsFiles: '**/test-lint.xml'
+      testRunTitle: 'Lint Results'
 
-  - script: pipenv install --dev --deploy --system
-    displayName: 'Install project using pipenv'
+  - script: sudo swapoff -a
+    displayName: 'Disable swap memory'
 
-  - script: python3 -m flake8 --format junit-xml --output-file test-lint.xml
-    displayName: 'Run linter'
+  - script: |
+      docker exec snekbox_test /bin/ash -c \
+        'pipenv run coverage run -m xmlrunner'
+    displayName: 'Run Unit Tests'
 
   - task: PublishTestResults@2
     condition: succeededOrFailed()
+    displayName: 'Publish Test Results'
+    inputs:
+      testResultsFiles: '**/TEST-*.xml'
+      testRunTitle: 'Test Results'
+
+  - script: |
+      docker exec snekbox_test /bin/ash -c \
+        'pipenv run coverage xml'
+    displayName: 'Generate Coverage Report'
+
+  - task: PublishCodeCoverageResults@1
+    displayName: 'Publish Coverage Results'
+    condition: succeededOrFailed()
     inputs:
-      testResultsFiles: '**/test-*.xml'
-      testRunTitle: 'Snekbox Flake8 Lint Results'
+      codeCoverageTool: Cobertura
+      summaryFileLocation: '**/coverage.xml'
 
 - job: build
   displayName: 'Build'
diff --git a/docker-compose.yml b/docker-compose.yml
index 1fe8e39..d071a71 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,3 +6,6 @@ services:
     image: pythondiscord/snekbox:latest
     network_mode: "host"
     init: true
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
diff --git a/docker/venv.Dockerfile b/docker/venv.Dockerfile
index 61aba58..85188fd 100644
--- a/docker/venv.Dockerfile
+++ b/docker/venv.Dockerfile
@@ -1,5 +1,6 @@
 FROM pythondiscord/snekbox-base:latest
 
+ARG DEV
 ENV PIP_NO_CACHE_DIR=false \
     PIPENV_DONT_USE_PYENV=1 \
     PIPENV_HIDE_EMOJIS=1 \
@@ -9,4 +10,4 @@ ENV PIP_NO_CACHE_DIR=false \
 COPY Pipfile Pipfile.lock /snekbox/
 WORKDIR /snekbox
 
-RUN pipenv sync
+RUN if [ -n "${DEV}" ]; pipenv sync --dev; then pipenv sync; fi
diff --git a/scripts/.profile b/scripts/.profile
new file mode 100644
index 0000000..bff260d
--- /dev/null
+++ b/scripts/.profile
@@ -0,0 +1,32 @@
+nsjpy() {
+    local MEM_MAX=52428800
+
+    # All arguments except the last are considered to be for NsJail, not Python.
+    local nsj_args=""
+    while [ "$#" -gt 1 ]; do
+        nsj_args="${nsj_args:+${nsj_args} }$1"
+        shift
+    done
+
+    # Set up cgroups and disable memory swapping.
+    mkdir -p /sys/fs/cgroup/pids/NSJAIL
+    mkdir -p /sys/fs/cgroup/memory/NSJAIL
+    echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.limit_in_bytes
+    echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.memsw.limit_in_bytes
+
+    nsjail \
+        -Mo \
+        --rlimit_as 700 \
+        --chroot / \
+        -E LANG=en_US.UTF-8 \
+        -R/usr -R/lib -R/lib64 \
+        --user nobody \
+        --group nogroup \
+        --time_limit 2 \
+        --disable_proc \
+        --iface_no_lo \
+        --cgroup_pids_max=1 \
+        --cgroup_mem_max="${MEM_MAX}" \
+        $nsj_args -- \
+        /snekbox/.venv/bin/python3 -Iq -c "$@"
+}
diff --git a/scripts/dev.sh b/scripts/dev.sh
new file mode 100755
index 0000000..097690b
--- /dev/null
+++ b/scripts/dev.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env sh
+
+# Sets up a development environment and runs a shell in a docker container.
+# Usage: dev.sh [--build [--clean]] [ash_args ...]
+
+if [ "$1" = "--build" ]; then
+    shift
+    printf "Building pythondiscord/snekbox-venv:dev..."
+
+    docker build \
+        -t pythondiscord/snekbox-venv:dev \
+        -f docker/venv.Dockerfile \
+        --build-arg DEV=1 \
+        -q \
+        . \
+        >/dev/null \
+    && printf " done!\n" || exit "$?"
+
+    if [ "$1" = "--clean" ]; then
+        shift
+        dangling_imgs=$(docker images -f "dangling=true" -q)
+
+        if [ -n "${dangling_imgs}" ]; then
+            printf "Removing dangling images..."
+
+            docker rmi $dangling_imgs >/dev/null \
+            && printf " done!\n" || exit "$?"
+        fi
+    fi
+fi
+
+# Keep the container up in the background so it doesn't have to be restarted
+# for the ownership fix.
+# The volume is mounted to same the path in the container as the source
+# directory on the host to ensure coverage can find the source files.
+docker run \
+    -td \
+    --name snekbox_test \
+    --privileged \
+    --network host \
+    -h pdsnk-dev \
+    -e PYTHONDONTWRITEBYTECODE=1 \
+    -e PIPENV_PIPFILE="/snekbox/Pipfile" \
+    -e ENV="${PWD}/scripts/.profile" \
+    -v "${PWD}":"${PWD}" \
+    -w "${PWD}"\
+    --entrypoint /bin/ash \
+    pythondiscord/snekbox-venv:dev \
+    >/dev/null \
+
+# Execute the given command(s)
+docker exec -it snekbox_test /bin/ash "$@"
+
+# Fix ownership of coverage file
+# BusyBox doesn't support --reference for chown
+docker exec \
+    -it \
+    -e CWD="${PWD}" \
+    snekbox_test \
+    /bin/ash \
+    -c 'chown "$(stat -c "%u:%g" "${CWD}")" "${CWD}/.coverage"'
+
+docker rm -f snekbox_test >/dev/null # Stop and remove the container
diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py
index b1dc34d..d980f09 100644
--- a/snekbox/nsjail.py
+++ b/snekbox/nsjail.py
@@ -5,7 +5,9 @@ import subprocess
 import sys
 import textwrap
 from pathlib import Path
+from subprocess import CompletedProcess
 from tempfile import NamedTemporaryFile
+from typing import Iterable
 
 from snekbox import DEBUG
 
@@ -13,7 +15,7 @@ log = logging.getLogger(__name__)
 
 # [level][timestamp][PID]? function_signature:line_no? message
 LOG_PATTERN = re.compile(
-    r"\[(?P<level>(I)|[WEF])\]\[.+?\](?(2)|(?P<func>\[\d+\] .+?:\d+ )) ?(?P<msg>.+)"
+    r"\[(?P<level>(I)|[DWEF])\]\[.+?\](?(2)|(?P<func>\[\d+\] .+?:\d+ )) ?(?P<msg>.+)"
 )
 LOG_BLACKLIST = ("Process will be ",)
 
@@ -22,6 +24,7 @@ CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL")
 CGROUP_MEMORY_PARENT = Path("/sys/fs/cgroup/memory/NSJAIL")
 
 NSJAIL_PATH = os.getenv("NSJAIL_PATH", "/usr/sbin/nsjail")
+MEM_MAX = 52428800
 
 
 class NsJail:
@@ -57,14 +60,30 @@ class NsJail:
 
         NsJail doesn't do this automatically because it requires privileges NsJail usually doesn't
         have.
+
+        Disables memory swapping.
         """
         pids.mkdir(parents=True, exist_ok=True)
         mem.mkdir(parents=True, exist_ok=True)
 
+        # Swap limit cannot be set to a value lower than memory.limit_in_bytes.
+        # Therefore, this must be set first.
+        (mem / "memory.limit_in_bytes").write_text(str(MEM_MAX), encoding="utf-8")
+
+        try:
+            # Swap limit is specified as the sum of the memory and swap limits.
+            (mem / "memory.memsw.limit_in_bytes").write_text(str(MEM_MAX), encoding="utf-8")
+        except PermissionError:
+            log.warning(
+                "Failed to set the memory swap limit for the cgroup. "
+                "This is probably because CONFIG_MEMCG_SWAP or CONFIG_MEMCG_SWAP_ENABLED is unset. "
+                "Please ensure swap memory is disabled on the system."
+            )
+
     @staticmethod
-    def _parse_log(log_file):
+    def _parse_log(log_lines: Iterable[str]):
         """Parse and log NsJail's log messages."""
-        for line in log_file.read().decode("UTF-8").splitlines():
+        for line in log_lines:
             match = LOG_PATTERN.fullmatch(line)
             if match is None:
                 log.warning(f"Failed to parse log line '{line}'")
@@ -91,7 +110,7 @@ class NsJail:
                 # Treat fatal as error.
                 log.error(msg)
 
-    def python3(self, code: str) -> subprocess.CompletedProcess:
+    def python3(self, code: str) -> CompletedProcess:
         """Execute Python 3 code in an isolated environment and return the completed process."""
         with NamedTemporaryFile() as nsj_log:
             args = (
@@ -100,13 +119,13 @@ class NsJail:
                 "--chroot", "/",
                 "-E", "LANG=en_US.UTF-8",
                 "-R/usr", "-R/lib", "-R/lib64",
-                "--user", "nobody",
-                "--group", "nogroup",
+                "--user", "65534",  # nobody
+                "--group", "65534",  # nobody/nogroup
                 "--time_limit", "2",
                 "--disable_proc",
                 "--iface_no_lo",
                 "--log", nsj_log.name,
-                "--cgroup_mem_max=52428800",
+                f"--cgroup_mem_max={MEM_MAX}",
                 "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent),
                 "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name,
                 "--cgroup_pids_max=1",
@@ -129,8 +148,13 @@ class NsJail:
                     text=True
                 )
             except ValueError:
-                return subprocess.CompletedProcess(args, None, "ValueError: embedded null byte", "")
+                return CompletedProcess(args, None, "ValueError: embedded null byte", None)
+
+            log_lines = nsj_log.read().decode("utf-8").splitlines()
+            if not log_lines and result.returncode == 255:
+                # NsJail probably failed to parse arguments so log output will still be in stdout
+                log_lines = result.stdout.splitlines()
 
-            self._parse_log(nsj_log)
+            self._parse_log(log_lines)
 
         return result
diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py
new file mode 100644
index 0000000..bb176d9
--- /dev/null
+++ b/tests/test_nsjail.py
@@ -0,0 +1,124 @@
+import logging
+import unittest
+from textwrap import dedent
+
+from snekbox.nsjail import MEM_MAX, NsJail
+
+
+class NsJailTests(unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+
+        self.nsjail = NsJail()
+        self.nsjail.DEBUG = False
+        self.logger = logging.getLogger("snekbox.nsjail")
+
+    def test_print_returns_0(self):
+        result = self.nsjail.python3("print('test')")
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(result.stdout, "test\n")
+        self.assertEqual(result.stderr, None)
+
+    def test_timeout_returns_137(self):
+        code = dedent("""
+            while True:
+                pass
+        """).strip()
+
+        with self.assertLogs(self.logger) as log:
+            result = self.nsjail.python3(code)
+
+        self.assertEqual(result.returncode, 137)
+        self.assertEqual(result.stdout, "")
+        self.assertEqual(result.stderr, None)
+        self.assertIn("run time >= time limit", "\n".join(log.output))
+
+    def test_memory_returns_137(self):
+        # Add a kilobyte just to be safe.
+        code = dedent(f"""
+            x = ' ' * {MEM_MAX + 1000}
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 137)
+        self.assertEqual(result.stdout, "")
+        self.assertEqual(result.stderr, None)
+
+    def test_subprocess_resource_unavailable(self):
+        code = dedent("""
+            import subprocess
+            print(subprocess.check_output('kill -9 6', shell=True).decode())
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Resource temporarily unavailable", result.stdout)
+        self.assertEqual(result.stderr, None)
+
+    def test_read_only_file_system(self):
+        code = dedent("""
+            open('hello', 'w').write('world')
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Read-only file system", result.stdout)
+        self.assertEqual(result.stderr, None)
+
+    def test_forkbomb_resource_unavailable(self):
+        code = dedent("""
+            import os
+            while 1:
+                os.fork()
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Resource temporarily unavailable", result.stdout)
+        self.assertEqual(result.stderr, None)
+
+    def test_sigsegv_returns_139(self):  # In honour of Juan.
+        code = dedent("""
+            import ctypes
+            ctypes.string_at(0)
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 139)
+        self.assertEqual(result.stdout, "")
+        self.assertEqual(result.stderr, None)
+
+    def test_null_byte_value_error(self):
+        result = self.nsjail.python3("\0")
+        self.assertEqual(result.returncode, None)
+        self.assertEqual(result.stdout, "ValueError: embedded null byte")
+        self.assertEqual(result.stderr, None)
+
+    def test_log_parser(self):
+        log_lines = (
+            "[D][2019-06-22T20:07:00+0000][16] void foo::bar()():100 This is a debug message.",
+            "[I][2019-06-22T20:07:48+0000] pid=20 ([STANDALONE MODE]) "
+            "exited with status: 2, (PIDs left: 0)",
+            "[W][2019-06-22T20:06:04+0000][14] void cmdline::logParams(nsjconf_t*)():250 "
+            "Process will be UID/EUID=0 in the global user namespace, and will have user "
+            "root-level access to files",
+            "[W][2019-06-22T20:07:00+0000][16] void foo::bar()():100 This is a warning!",
+            "[E][2019-06-22T20:07:00+0000][16] bool "
+            "cmdline::setupArgv(nsjconf_t*, int, char**, int)():316 No command-line provided",
+            "[F][2019-06-22T20:07:00+0000][16] int main(int, char**)():204 "
+            "Couldn't parse cmdline options",
+            "Invalid Line"
+        )
+
+        with self.assertLogs(self.logger, logging.DEBUG) as log:
+            self.nsjail._parse_log(log_lines)
+
+        self.assertIn("DEBUG:snekbox.nsjail:This is a debug message.", log.output)
+        self.assertIn("ERROR:snekbox.nsjail:Couldn't parse cmdline options", log.output)
+        self.assertIn("ERROR:snekbox.nsjail:No command-line provided", log.output)
+        self.assertIn("WARNING:snekbox.nsjail:Failed to parse log line 'Invalid Line'", log.output)
+        self.assertIn("WARNING:snekbox.nsjail:This is a warning!", log.output)
+        self.assertIn(
+            "INFO:snekbox.nsjail:pid=20 ([STANDALONE MODE]) exited with status: 2, (PIDs left: 0)",
+            log.output
+        )
diff --git a/tests/test_snekbox.py b/tests/test_snekbox.py
deleted file mode 100644
index 46319d6..0000000
--- a/tests/test_snekbox.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import unittest
-
-from snekbox.nsjail import NsJail
-
-nsjail = NsJail()
-
-
-class SnekTests(unittest.TestCase):
-    def test_nsjail(self):
-        result = nsjail.python3("print('test')")
-        self.assertEquals(result.strip(), "test")
-
-    # def test_memory_error(self):
-    #     code = ("x = "*"\n"
-    #             "while True:\n"
-    #             "    x = x * 99\n")
-    #     result = nsjail.python3(code)
-    #     self.assertEquals(result.strip(), "timed out or memory limit exceeded")
-
-    def test_timeout(self):
-        code = (
-            "x = '*'\n"
-            "while True:\n"
-            "    try:\n"
-            "        x = x * 99\n"
-            "    except:\n"
-            "        continue\n"
-        )
-
-        result = nsjail.python3(code)
-        self.assertEquals(result.strip(), "timed out or memory limit exceeded")
-
-    def test_kill(self):
-        code = ("import subprocess\n"
-                "print(subprocess.check_output('kill -9 6', shell=True).decode())")
-        result = nsjail.python3(code)
-        if "ModuleNotFoundError" in result.strip():
-            self.assertIn("ModuleNotFoundError", result.strip())
-        else:
-            self.assertIn("(PIDs left: 0)", result.strip())
-
-    def test_forkbomb(self):
-        code = ("import os\n"
-                "while 1:\n"
-                "    os.fork()")
-        result = nsjail.python3(code)
-        self.assertIn("Resource temporarily unavailable", result.strip())
-
-    def test_juan_golf(self):  # in honour of Juan
-        code = ("func = lambda: None\n"
-                "CodeType = type(func.__code__)\n"
-                "bytecode = CodeType(0,1,0,0,0,b'',(),(),(),'','',1,b'')\n"
-                "exec(bytecode)")
-
-        result = nsjail.python3(code)
-        self.assertEquals("unknown error, code: 111", result.strip())