From 2c843101843b975ece546b8921d53b3dd4e6974d Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Thu, 6 Jun 2019 16:54:33 -0700
Subject: Create shell script for building a dev image and running a shell

* Put scripts in a new scripts folder
---
 scripts/.profile | 25 +++++++++++++++++++++++++
 scripts/dev.sh   | 45 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 scripts/.profile
 create mode 100755 scripts/dev.sh

(limited to 'scripts')

diff --git a/scripts/.profile b/scripts/.profile
new file mode 100644
index 0000000..415e4f6
--- /dev/null
+++ b/scripts/.profile
@@ -0,0 +1,25 @@
+nsjpy() {
+    local nsj_args=""
+    while [ "$#" -gt 1 ]; do
+        nsj_args="${nsj_args:+${nsj_args} }$1"
+        shift
+    done
+
+    mkdir -p /sys/fs/cgroup/pids/NSJAIL
+    mkdir -p /sys/fs/cgroup/memory/NSJAIL
+    nsjail \
+        -Mo \
+        --rlimit_as 700 \
+        --chroot / \
+        -E LANG=en_US.UTF-8 \
+        -R/usr -R/lib -R/lib64 \
+        --user nobody \
+        --group nogroup \
+        --time_limit 2 \
+        --disable_proc \
+        --iface_no_lo \
+        --cgroup_pids_max=1 \
+        --cgroup_mem_max=52428800 \
+        $nsj_args -- \
+        /snekbox/.venv/bin/python3 -Iq -c "$@"
+}
diff --git a/scripts/dev.sh b/scripts/dev.sh
new file mode 100755
index 0000000..490021f
--- /dev/null
+++ b/scripts/dev.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env sh
+
+# Sets up a development environment and runs a shell in a docker container.
+# Usage: dev.sh [--build [--clean]] [ash_args ...]
+
+if [ "$1" = "--build" ]; then
+    shift
+    printf "Building pythondiscord/snekbox-venv:dev..."
+
+    docker build \
+        -t pythondiscord/snekbox-venv:dev \
+        -f docker/venv.Dockerfile \
+        --build-arg DEV=1 \
+        -q \
+        . \
+        >/dev/null \
+    && printf " done!\n" || exit "$?"
+
+    if [ "$1" = "--clean" ]; then
+        shift
+        dangling_imgs=$(docker images -f "dangling=true" -q)
+
+        if [ -n "${dangling_imgs}" ]; then
+            printf "Removing dangling images..."
+
+            docker rmi $dangling_imgs >/dev/null \
+            && printf " done!\n" || exit "$?"
+        fi
+    fi
+fi
+
+docker run \
+    -it \
+    --rm \
+    --privileged \
+    --network host \
+    -h pdsnk-dev \
+    -e PYTHONDONTWRITEBYTECODE=1 \
+    -e PIPENV_PIPFILE="/snekbox/Pipfile" \
+    -e ENV="/snekbox-local/scripts/.profile" \
+    -v "${PWD}":/snekbox-local \
+    -w "/snekbox-local" \
+    --entrypoint /bin/ash \
+    pythondiscord/snekbox-venv:dev \
+    "$@"
-- 
cgit v1.2.3


From c1a6440899ced2f3f787352cd1d3ea1f49e520ee Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Thu, 20 Jun 2019 16:25:28 -0700
Subject: Fix ownership of coverage file

When coverage runs in a container, it is ran under root so the resulting
coverage file is owned by root. chown is used to change ownership to
be the same as the folder it is in.
---
 scripts/dev.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'scripts')

diff --git a/scripts/dev.sh b/scripts/dev.sh
index 490021f..6ebae71 100755
--- a/scripts/dev.sh
+++ b/scripts/dev.sh
@@ -31,7 +31,7 @@ fi
 
 docker run \
     -it \
-    --rm \
+    --name snekbox_test \
     --privileged \
     --network host \
     -h pdsnk-dev \
@@ -43,3 +43,12 @@ docker run \
     --entrypoint /bin/ash \
     pythondiscord/snekbox-venv:dev \
     "$@"
+
+# Fix ownership of coverage file
+docker start snekbox_test >/dev/null
+docker exec \
+    -it \
+    snekbox_test \
+    /bin/ash \
+    -c 'chown "$(stat -c "%u:%g" "/snekbox-local")" /snekbox-local/.coverage'
+docker rm -f snekbox_test >/dev/null
-- 
cgit v1.2.3


From 495baa3045c63040f460538e94eaaed6a6499fba Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Fri, 21 Jun 2019 19:35:57 -0700
Subject: Fix coverage not finding sources

* Mount volume to the same path as the source directory on the host
* Keep the container up in the background so it doesn't have to be
  restarted or the ownership fix
---
 scripts/dev.sh | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

(limited to 'scripts')

diff --git a/scripts/dev.sh b/scripts/dev.sh
index 6ebae71..097690b 100755
--- a/scripts/dev.sh
+++ b/scripts/dev.sh
@@ -29,26 +29,35 @@ if [ "$1" = "--build" ]; then
     fi
 fi
 
+# Keep the container up in the background so it doesn't have to be restarted
+# for the ownership fix.
+# The volume is mounted to same the path in the container as the source
+# directory on the host to ensure coverage can find the source files.
 docker run \
-    -it \
+    -td \
     --name snekbox_test \
     --privileged \
     --network host \
     -h pdsnk-dev \
     -e PYTHONDONTWRITEBYTECODE=1 \
     -e PIPENV_PIPFILE="/snekbox/Pipfile" \
-    -e ENV="/snekbox-local/scripts/.profile" \
-    -v "${PWD}":/snekbox-local \
-    -w "/snekbox-local" \
+    -e ENV="${PWD}/scripts/.profile" \
+    -v "${PWD}":"${PWD}" \
+    -w "${PWD}"\
     --entrypoint /bin/ash \
     pythondiscord/snekbox-venv:dev \
-    "$@"
+    >/dev/null \
+
+# Execute the given command(s)
+docker exec -it snekbox_test /bin/ash "$@"
 
 # Fix ownership of coverage file
-docker start snekbox_test >/dev/null
+# BusyBox doesn't support --reference for chown
 docker exec \
     -it \
+    -e CWD="${PWD}" \
     snekbox_test \
     /bin/ash \
-    -c 'chown "$(stat -c "%u:%g" "/snekbox-local")" /snekbox-local/.coverage'
-docker rm -f snekbox_test >/dev/null
+    -c 'chown "$(stat -c "%u:%g" "${CWD}")" "${CWD}/.coverage"'
+
+docker rm -f snekbox_test >/dev/null # Stop and remove the container
-- 
cgit v1.2.3


From 158915a953879639722ab3bc1074fec7276117ba Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Wed, 26 Jun 2019 23:00:39 -0700
Subject: Disable memory swapping and add a memory limit test

If memory swapping was enabled locally, the memory test would fail.
Explicitly disabling swapping also removes reliance on the assumption
that it'll be disabled in production.

* Add a constant for the maximum memory
* Simplify the timeout test; it'd otherwise first run out of memory now
---
 scripts/.profile     |  9 ++++++++-
 snekbox/nsjail.py    | 14 +++++++++++++-
 tests/test_nsjail.py | 19 +++++++++++++------
 3 files changed, 34 insertions(+), 8 deletions(-)

(limited to 'scripts')

diff --git a/scripts/.profile b/scripts/.profile
index 415e4f6..bff260d 100644
--- a/scripts/.profile
+++ b/scripts/.profile
@@ -1,12 +1,19 @@
 nsjpy() {
+    local MEM_MAX=52428800
+
+    # All arguments except the last are considered to be for NsJail, not Python.
     local nsj_args=""
     while [ "$#" -gt 1 ]; do
         nsj_args="${nsj_args:+${nsj_args} }$1"
         shift
     done
 
+    # Set up cgroups and disable memory swapping.
     mkdir -p /sys/fs/cgroup/pids/NSJAIL
     mkdir -p /sys/fs/cgroup/memory/NSJAIL
+    echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.limit_in_bytes
+    echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.memsw.limit_in_bytes
+
     nsjail \
         -Mo \
         --rlimit_as 700 \
@@ -19,7 +26,7 @@ nsjpy() {
         --disable_proc \
         --iface_no_lo \
         --cgroup_pids_max=1 \
-        --cgroup_mem_max=52428800 \
+        --cgroup_mem_max="${MEM_MAX}" \
         $nsj_args -- \
         /snekbox/.venv/bin/python3 -Iq -c "$@"
 }
diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py
index b68b0b9..b9c4fc7 100644
--- a/snekbox/nsjail.py
+++ b/snekbox/nsjail.py
@@ -24,6 +24,7 @@ CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL")
 CGROUP_MEMORY_PARENT = Path("/sys/fs/cgroup/memory/NSJAIL")
 
 NSJAIL_PATH = os.getenv("NSJAIL_PATH", "/usr/sbin/nsjail")
+MEM_MAX = 52428800
 
 
 class NsJail:
@@ -59,10 +60,21 @@ class NsJail:
 
         NsJail doesn't do this automatically because it requires privileges NsJail usually doesn't
         have.
+
+        Disables memory swapping.
         """
         pids.mkdir(parents=True, exist_ok=True)
         mem.mkdir(parents=True, exist_ok=True)
 
+        # Swap limit cannot be set to a value lower than memory.limit_in_bytes.
+        # Therefore, this must be set first.
+        with (mem / "memory.limit_in_bytes").open("w", encoding="utf=8") as f:
+            f.write(str(MEM_MAX))
+
+        # Swap limit is specified as the sum of the memory and swap limits.
+        with (mem / "memory.memsw.limit_in_bytes").open("w", encoding="utf=8") as f:
+            f.write(str(MEM_MAX))
+
     @staticmethod
     def _parse_log(log_lines: Iterable[str]):
         """Parse and log NsJail's log messages."""
@@ -108,7 +120,7 @@ class NsJail:
                 "--disable_proc",
                 "--iface_no_lo",
                 "--log", nsj_log.name,
-                "--cgroup_mem_max=52428800",
+                f"--cgroup_mem_max={MEM_MAX}",
                 "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent),
                 "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name,
                 "--cgroup_pids_max=1",
diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py
index e3b8eb3..f1a60e6 100644
--- a/tests/test_nsjail.py
+++ b/tests/test_nsjail.py
@@ -2,7 +2,7 @@ import logging
 import unittest
 from textwrap import dedent
 
-from snekbox.nsjail import NsJail
+from snekbox.nsjail import MEM_MAX, NsJail
 
 
 class NsJailTests(unittest.TestCase):
@@ -21,12 +21,8 @@ class NsJailTests(unittest.TestCase):
 
     def test_timeout_returns_137(self):
         code = dedent("""
-            x = '*'
             while True:
-                try:
-                    x = x * 99
-                except:
-                    continue
+                pass
         """).strip()
 
         with self.assertLogs(self.logger) as log:
@@ -37,6 +33,17 @@ class NsJailTests(unittest.TestCase):
         self.assertEqual(result.stderr, None)
         self.assertIn("run time >= time limit", "\n".join(log.output))
 
+    def test_memory_returns_137(self):
+        # Add a kilobyte just to be safe.
+        code = dedent(f"""
+            x = ' ' * {MEM_MAX + 1000}
+        """).strip()
+
+        result = self.nsjail.python3(code)
+        self.assertEqual(result.returncode, 137)
+        self.assertEqual(result.stdout, "")
+        self.assertEqual(result.stderr, None)
+
     def test_subprocess_resource_unavailable(self):
         code = dedent("""
             import subprocess
-- 
cgit v1.2.3


From 96b5a70dc50ce9edce6439967ce35385ad6de22f Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 12:23:28 -0700
Subject: CI: move check shell script to a separate file

---
 azure-pipelines.yml          | 35 ++++-------------------------------
 scripts/check_dockerfiles.sh | 31 +++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 31 deletions(-)
 create mode 100755 scripts/check_dockerfiles.sh

(limited to 'scripts')

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 591f87f..bbca0b7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -96,37 +96,10 @@ jobs:
           dockerRegistryEndpoint: 'DockerHub'
           command: 'login'
 
-      - script: |
-          REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0"
-          echo "Retrieving previous build's commit using $REQUEST_URL"
-          RESPONSE="$(curl -sSL "${REQUEST_URL}")"
-
-          if [[ $BUILD_REASON = "PullRequest" ]]; then
-            PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-            if [[ -z $PREV_COMMIT ]]; then
-              echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch."
-              PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH"
-            fi
-          else
-            PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-          fi
-
-          if [[ -n $PREV_COMMIT ]]; then
-            echo "Using $PREV_COMMIT to compare diffs."
-
-            if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then
-              echo "No changes detected in docker/base.Dockerfile. The base image will not be built."
-              echo "##vso[task.setvariable variable=BASE_CHANGED]false"
-            fi
-
-            if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then
-              echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built."
-              echo "##vso[task.setvariable variable=VENV_CHANGED]false"
-            fi
-          else
-            echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built."
-          fi
-        displayName: 'Check Changed Files'
+      - task: ShellScript@2
+        displayName: 'Check If Images Need to Be Built'
+        inputs:
+          scriptPath: scripts/check_dockerfiles.sh
 
       - task: Docker@2
         displayName: 'Build Base Image'
diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
new file mode 100755
index 0000000..07e76f8
--- /dev/null
+++ b/scripts/check_dockerfiles.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0"
+echo "Retrieving previous build's commit using $REQUEST_URL"
+RESPONSE="$(curl -sSL "${REQUEST_URL}")"
+
+if [[ $BUILD_REASON = "PullRequest" ]]; then
+    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
+    if [[ -z $PREV_COMMIT ]]; then
+        echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch."
+        PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH"
+    fi
+else
+    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
+fi
+
+if [[ -n $PREV_COMMIT ]]; then
+    echo "Using $PREV_COMMIT to compare diffs."
+
+    if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then
+        echo "No changes detected in docker/base.Dockerfile. The base image will not be built."
+        echo "##vso[task.setvariable variable=BASE_CHANGED]false"
+    fi
+
+    if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then
+        echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built."
+        echo "##vso[task.setvariable variable=VENV_CHANGED]false"
+    fi
+else
+    echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built."
+fi
-- 
cgit v1.2.3


From c50a28524733a3ad4ee9767c1a0a1bab059489f1 Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 14:38:46 -0700
Subject: CI: refactor script & pull base when possible

* Move script's execution to the test job
* Use output variables
* Use jq instead of regex for parsing JSON responses from API
* Wrap to 80 columns
* Make more robust by checking for command success
---
 azure-pipelines.yml          | 25 +++++++----
 scripts/check_dockerfiles.sh | 98 ++++++++++++++++++++++++++++++++++----------
 2 files changed, 93 insertions(+), 30 deletions(-)

(limited to 'scripts')

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index bbca0b7..7467f3b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -8,8 +8,15 @@ jobs:
       vmImage: 'ubuntu-16.04'
 
     steps:
+      - task: ShellScript@2
+        displayName: 'Check If Images Need to Be Built'
+        name: check
+        inputs:
+          scriptPath: scripts/check_dockerfiles.sh
+
       - task: Docker@2
         displayName: 'Build Base Image'
+        condition: and(succeeded(), ne(variables['check.BASE_PULL'], True))
         inputs:
           command: build
           repository: pythondiscord/snekbox-base
@@ -85,8 +92,9 @@ jobs:
     dependsOn: test
 
     variables:
-      BASE_CHANGED: true
-      VENV_CHANGED: true
+      BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ]
+      VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ]
+      BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ]
 
     steps:
       - task: Docker@1
@@ -96,14 +104,14 @@ jobs:
           dockerRegistryEndpoint: 'DockerHub'
           command: 'login'
 
-      - task: ShellScript@2
-        displayName: 'Check If Images Need to Be Built'
-        inputs:
-          scriptPath: scripts/check_dockerfiles.sh
-
       - task: Docker@2
         displayName: 'Build Base Image'
-        condition: and(succeeded(), eq(variables.BASE_CHANGED, True))
+        condition: >
+          and(
+            succeeded(),
+            ne(variables.BASE_PULL, True),
+            eq(variables.BASE_CHANGED, True)
+          )
         inputs:
           command: build
           repository: pythondiscord/snekbox-base
@@ -141,6 +149,7 @@ jobs:
           and(
             succeeded(),
             ne(variables['Build.Reason'], 'PullRequest'),
+            ne(variables.BASE_PULL, True),
             eq(variables.BASE_CHANGED, True)
           )
         inputs:
diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
index 07e76f8..015fa41 100755
--- a/scripts/check_dockerfiles.sh
+++ b/scripts/check_dockerfiles.sh
@@ -1,31 +1,85 @@
 #!/usr/bin/env bash
 
-REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0"
-echo "Retrieving previous build's commit using $REQUEST_URL"
-RESPONSE="$(curl -sSL "${REQUEST_URL}")"
-
-if [[ $BUILD_REASON = "PullRequest" ]]; then
-    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-    if [[ -z $PREV_COMMIT ]]; then
-        echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch."
-        PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH"
-    fi
-else
-    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-fi
+set -euo pipefail
+exec 3>&1 # New file descriptor to stdout
+
+BASE_URL="https://dev.azure.com/\
+python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\
+queryOrder=finishTimeDescending&\
+resultFilter=succeeded&\
+\$top=1&\
+repositoryType=${BUILD_REPOSITORY_PROVIDER}&\
+repositoryId=${BUILD_REPOSITORY_NAME}&\
+api-version=5.0"
+
+get_build() {
+    set -e # Poor Ubuntu LTS doesn't have Bash 4.4's inherit_errexit
+
+    local branch="${1:?"get_build: argument 1 'branch' is unset"}"
+    local url="${BASE_URL}&branchName=${branch}"
+
+    printf '%s\n' "Retrieving the latest successful build using ${url}" >&3
 
-if [[ -n $PREV_COMMIT ]]; then
-    echo "Using $PREV_COMMIT to compare diffs."
+    local response
+    response="$(curl -sSL "${url}")"
 
-    if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then
-        echo "No changes detected in docker/base.Dockerfile. The base image will not be built."
-        echo "##vso[task.setvariable variable=BASE_CHANGED]false"
+    if [[ -z "${response}" ]] \
+        || ! printf '%s' "${response}" | jq -re '.count'
+    then
+        return 1
+    else
+        printf '%s' "${response}"
     fi
+}
 
-    if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then
-        echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built."
-        echo "##vso[task.setvariable variable=VENV_CHANGED]false"
+# Get the previous commit
+if [[ "${BUILD_REASON}" = "PullRequest" ]]; then
+    if ! prev_commit="$(
+            get_build "${BUILD_SOURCEBRANCH}" \
+            | jq -re '.value[0].triggerInfo."pr.sourceSha"'
+        )"
+    then
+        echo \
+            "Could not retrieve the previous build's commit." \
+            "Falling back to the head of the target branch."
+
+        prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}"
     fi
+elif ! prev_commit="$(
+        get_build "${BUILD_SOURCEBRANCH}" \
+        | jq -re '.value[0].sourceVersion'
+    )"
+then
+    echo \
+        "No previous build was found." \
+        "Either the previous build is too old and was deleted" \
+        "or the branch was empty before this build." \
+        "All images will be built."
+    exit 0
+fi
+
+# Compare diffs
+head="$(git rev-parse HEAD)"
+printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}."
+
+if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then
+    echo "No changes detected in docker/base.Dockerfile."
+    echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False"
 else
-    echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built."
+    # Always rebuild the venv if the base changes.
+    exit 0
+fi
+
+if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then
+    echo "No changes detected in docker/venv.Dockerfile or the Pipfiles."
+    echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False"
+elif master_commit="$(
+        get_build "refs/heads/master" \
+        | jq -re '.value[0].sourceVersion'
+    )" \
+    && git diff --quiet "${master_commit}" -- docker/base.Dockerfile
+then
+    # Though base image hasn't changed, it's still needed to build the venv.
+    echo "Can pull base image from Docker Hub; no changes made since master."
+    echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True"
 fi
-- 
cgit v1.2.3


From 931ec33623e3b0b9c1d56621a677116be15108de Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 16:35:24 -0700
Subject: CI: ensure count of builds returned by the API is > 0

---
 scripts/check_dockerfiles.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'scripts')

diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
index 015fa41..c84c61f 100755
--- a/scripts/check_dockerfiles.sh
+++ b/scripts/check_dockerfiles.sh
@@ -24,7 +24,8 @@ get_build() {
     response="$(curl -sSL "${url}")"
 
     if [[ -z "${response}" ]] \
-        || ! printf '%s' "${response}" | jq -re '.count'
+        || ! count="$(printf '%s' "${response}" | jq -re '.count')" \
+        || (( "${count}" < 1 ))
     then
         return 1
     else
-- 
cgit v1.2.3


From 2f864a14a3f4d49d11801af45946dbd81e3e343f Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Tue, 30 Jul 2019 15:31:19 -0700
Subject: Add comments to Azure Pipelines YAML

* Replace some shorthand Docker command options with their full names
  for clarity
---
 azure-pipelines.yml | 29 ++++++++++++++++++++++++-----
 scripts/dev.sh      |  9 +++++----
 2 files changed, 29 insertions(+), 9 deletions(-)

(limited to 'scripts')

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 3b7c1dc..f7b8eb7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -14,13 +14,16 @@ jobs:
         inputs:
           scriptPath: scripts/check_dockerfiles.sh
 
-      # Without a login the following Docker build tasks won't add image tags
+      # Without a login the following Docker build tasks won't add image tags.
       - task: Docker@2
         displayName: 'Log into Docker Hub'
         inputs:
           command: login
           containerRegistry: DockerHubV2
 
+      # The venv image depends on this image. Build it if it can't be pulled
+      # from Docker Hub, which will be the case if the base Dockerfile has had
+      # changes.
       - task: Docker@2
         displayName: 'Build Base Image'
         condition: and(succeeded(), ne(variables['check.BASE_PULL'], True))
@@ -31,6 +34,7 @@ jobs:
           Dockerfile: docker/base.Dockerfile
           buildContext: .
 
+      # The dev image is never pushed and therefore is always built.
       - task: Docker@2
         displayName: 'Build Development Image'
         inputs:
@@ -41,18 +45,20 @@ jobs:
           buildContext: .
           arguments: --build-arg DEV=1
 
+      # The linter and all tests run inside this container.
       - script: |
           docker run \
-            -td \
+            --tty \
+            --detach \
             --name snekbox_test \
             --privileged \
             --network host \
-            -h pdsnk-dev \
+            --hostname pdsnk-dev \
             -e PYTHONDONTWRITEBYTECODE=1 \
             -e PIPENV_PIPFILE="/snekbox/Pipfile" \
             -e ENV="${PWD}/scripts/.profile" \
-            -v "${PWD}":"${PWD}" \
-            -w "${PWD}"\
+            --volume "${PWD}":"${PWD}" \
+            --workdir "${PWD}"\
             --entrypoint /bin/ash \
             pythondiscord/snekbox-venv:dev
         displayName: 'Start Container'
@@ -69,6 +75,7 @@ jobs:
           testResultsFiles: '**/test-lint.xml'
           testRunTitle: 'Lint Results'
 
+      # Memory limit tests would fail if this isn't disabled.
       - script: sudo swapoff -a
         displayName: 'Disable Swap Memory'
 
@@ -96,6 +103,9 @@ jobs:
           codeCoverageTool: Cobertura
           summaryFileLocation: '**/coverage.xml'
 
+  # When a pull request, only perform this job if images need to be built.
+  # It's always performed for non-PRs because the final image will always need
+  # to be built.
   - job: build
     displayName: 'Build'
     condition: >
@@ -109,6 +119,7 @@ jobs:
       )
     dependsOn: test
 
+    # coalesce() gives variables default values if they are null (i.e. unset).
     variables:
       BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ]
       VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ]
@@ -121,6 +132,9 @@ jobs:
           command: login
           containerRegistry: DockerHubV2
 
+      # Because this is the base image for the venv image, if the venv needs to
+      # be built, this base image must also be present. Build it if it has
+      # changed or can't be pulled from Docker Hub.
       - task: Docker@2
         displayName: 'Build Base Image'
         condition: >
@@ -139,6 +153,7 @@ jobs:
           Dockerfile: docker/base.Dockerfile
           buildContext: .
 
+      # Also build this image if base has changed - even if this image hasn't.
       - task: Docker@2
         displayName: 'Build Virtual Environment Image'
         condition: >
@@ -156,6 +171,7 @@ jobs:
           Dockerfile: docker/venv.Dockerfile
           buildContext: .
 
+      # Always build this image unless it's for a pull request.
       - task: Docker@2
         displayName: 'Build Final Image'
         condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
@@ -166,6 +182,9 @@ jobs:
           Dockerfile: docker/Dockerfile
           buildContext: .
 
+      # Push images only after they've all successfully been built.
+      # These have the same conditions as the build tasks. However, for safety,
+      # a condition for not being a pull request is added.
       - task: Docker@2
         displayName: 'Push Base Image'
         condition: >
diff --git a/scripts/dev.sh b/scripts/dev.sh
index 097690b..8f5b24f 100755
--- a/scripts/dev.sh
+++ b/scripts/dev.sh
@@ -34,16 +34,17 @@ fi
 # The volume is mounted to same the path in the container as the source
 # directory on the host to ensure coverage can find the source files.
 docker run \
-    -td \
+    --tty \
+    --detach \
     --name snekbox_test \
     --privileged \
     --network host \
-    -h pdsnk-dev \
+    --hostname pdsnk-dev \
     -e PYTHONDONTWRITEBYTECODE=1 \
     -e PIPENV_PIPFILE="/snekbox/Pipfile" \
     -e ENV="${PWD}/scripts/.profile" \
-    -v "${PWD}":"${PWD}" \
-    -w "${PWD}"\
+    --volume "${PWD}":"${PWD}" \
+    --workdir "${PWD}"\
     --entrypoint /bin/ash \
     pythondiscord/snekbox-venv:dev \
     >/dev/null \
-- 
cgit v1.2.3


From c1a786df0cc2811544e276436d1d713eed9f8a0f Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 4 Aug 2019 21:50:26 -0700
Subject: Use IDs for user and group in nsjpy alias

Reflects the changes in 7a7eca52019bf21d21cdffcf03cd9c5eacd8363b
---
 scripts/.profile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'scripts')

diff --git a/scripts/.profile b/scripts/.profile
index bff260d..bd46a17 100644
--- a/scripts/.profile
+++ b/scripts/.profile
@@ -20,8 +20,8 @@ nsjpy() {
         --chroot / \
         -E LANG=en_US.UTF-8 \
         -R/usr -R/lib -R/lib64 \
-        --user nobody \
-        --group nogroup \
+        --user 65534 \
+        --group 65534 \
         --time_limit 2 \
         --disable_proc \
         --iface_no_lo \
-- 
cgit v1.2.3