From 96b5a70dc50ce9edce6439967ce35385ad6de22f Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 12:23:28 -0700
Subject: CI: move check shell script to a separate file

---
 scripts/check_dockerfiles.sh | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100755 scripts/check_dockerfiles.sh

(limited to 'scripts/check_dockerfiles.sh')

diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
new file mode 100755
index 0000000..07e76f8
--- /dev/null
+++ b/scripts/check_dockerfiles.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0"
+echo "Retrieving previous build's commit using $REQUEST_URL"
+RESPONSE="$(curl -sSL "${REQUEST_URL}")"
+
+if [[ $BUILD_REASON = "PullRequest" ]]; then
+    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
+    if [[ -z $PREV_COMMIT ]]; then
+        echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch."
+        PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH"
+    fi
+else
+    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
+fi
+
+if [[ -n $PREV_COMMIT ]]; then
+    echo "Using $PREV_COMMIT to compare diffs."
+
+    if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then
+        echo "No changes detected in docker/base.Dockerfile. The base image will not be built."
+        echo "##vso[task.setvariable variable=BASE_CHANGED]false"
+    fi
+
+    if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then
+        echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built."
+        echo "##vso[task.setvariable variable=VENV_CHANGED]false"
+    fi
+else
+    echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built."
+fi
-- 
cgit v1.2.3


From c50a28524733a3ad4ee9767c1a0a1bab059489f1 Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 14:38:46 -0700
Subject: CI: refactor script & pull base when possible

* Move script's execution to the test job
* Use output variables
* Use jq instead of regex for parsing JSON responses from API
* Wrap to 80 columns
* Make more robust by checking for command success
---
 azure-pipelines.yml          | 25 +++++++----
 scripts/check_dockerfiles.sh | 98 ++++++++++++++++++++++++++++++++++----------
 2 files changed, 93 insertions(+), 30 deletions(-)

(limited to 'scripts/check_dockerfiles.sh')

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index bbca0b7..7467f3b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -8,8 +8,15 @@ jobs:
       vmImage: 'ubuntu-16.04'
 
     steps:
+      - task: ShellScript@2
+        displayName: 'Check If Images Need to Be Built'
+        name: check
+        inputs:
+          scriptPath: scripts/check_dockerfiles.sh
+
       - task: Docker@2
         displayName: 'Build Base Image'
+        condition: and(succeeded(), ne(variables['check.BASE_PULL'], True))
         inputs:
           command: build
           repository: pythondiscord/snekbox-base
@@ -85,8 +92,9 @@ jobs:
     dependsOn: test
 
     variables:
-      BASE_CHANGED: true
-      VENV_CHANGED: true
+      BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ]
+      VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ]
+      BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ]
 
     steps:
       - task: Docker@1
@@ -96,14 +104,14 @@ jobs:
           dockerRegistryEndpoint: 'DockerHub'
           command: 'login'
 
-      - task: ShellScript@2
-        displayName: 'Check If Images Need to Be Built'
-        inputs:
-          scriptPath: scripts/check_dockerfiles.sh
-
       - task: Docker@2
         displayName: 'Build Base Image'
-        condition: and(succeeded(), eq(variables.BASE_CHANGED, True))
+        condition: >
+          and(
+            succeeded(),
+            ne(variables.BASE_PULL, True),
+            eq(variables.BASE_CHANGED, True)
+          )
         inputs:
           command: build
           repository: pythondiscord/snekbox-base
@@ -141,6 +149,7 @@ jobs:
           and(
             succeeded(),
             ne(variables['Build.Reason'], 'PullRequest'),
+            ne(variables.BASE_PULL, True),
             eq(variables.BASE_CHANGED, True)
           )
         inputs:
diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
index 07e76f8..015fa41 100755
--- a/scripts/check_dockerfiles.sh
+++ b/scripts/check_dockerfiles.sh
@@ -1,31 +1,85 @@
 #!/usr/bin/env bash
 
-REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0"
-echo "Retrieving previous build's commit using $REQUEST_URL"
-RESPONSE="$(curl -sSL "${REQUEST_URL}")"
-
-if [[ $BUILD_REASON = "PullRequest" ]]; then
-    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-    if [[ -z $PREV_COMMIT ]]; then
-        echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch."
-        PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH"
-    fi
-else
-    PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')"
-fi
+set -euo pipefail
+exec 3>&1 # New file descriptor to stdout
+
+BASE_URL="https://dev.azure.com/\
+python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\
+queryOrder=finishTimeDescending&\
+resultFilter=succeeded&\
+\$top=1&\
+repositoryType=${BUILD_REPOSITORY_PROVIDER}&\
+repositoryId=${BUILD_REPOSITORY_NAME}&\
+api-version=5.0"
+
+get_build() {
+    set -e # Poor Ubuntu LTS doesn't have Bash 4.4's inherit_errexit
+
+    local branch="${1:?"get_build: argument 1 'branch' is unset"}"
+    local url="${BASE_URL}&branchName=${branch}"
+
+    printf '%s\n' "Retrieving the latest successful build using ${url}" >&3
 
-if [[ -n $PREV_COMMIT ]]; then
-    echo "Using $PREV_COMMIT to compare diffs."
+    local response
+    response="$(curl -sSL "${url}")"
 
-    if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then
-        echo "No changes detected in docker/base.Dockerfile. The base image will not be built."
-        echo "##vso[task.setvariable variable=BASE_CHANGED]false"
+    if [[ -z "${response}" ]] \
+        || ! printf '%s' "${response}" | jq -re '.count'
+    then
+        return 1
+    else
+        printf '%s' "${response}"
     fi
+}
 
-    if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then
-        echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built."
-        echo "##vso[task.setvariable variable=VENV_CHANGED]false"
+# Get the previous commit
+if [[ "${BUILD_REASON}" = "PullRequest" ]]; then
+    if ! prev_commit="$(
+            get_build "${BUILD_SOURCEBRANCH}" \
+            | jq -re '.value[0].triggerInfo."pr.sourceSha"'
+        )"
+    then
+        echo \
+            "Could not retrieve the previous build's commit." \
+            "Falling back to the head of the target branch."
+
+        prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}"
     fi
+elif ! prev_commit="$(
+        get_build "${BUILD_SOURCEBRANCH}" \
+        | jq -re '.value[0].sourceVersion'
+    )"
+then
+    echo \
+        "No previous build was found." \
+        "Either the previous build is too old and was deleted" \
+        "or the branch was empty before this build." \
+        "All images will be built."
+    exit 0
+fi
+
+# Compare diffs
+head="$(git rev-parse HEAD)"
+printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}."
+
+if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then
+    echo "No changes detected in docker/base.Dockerfile."
+    echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False"
 else
-    echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built."
+    # Always rebuild the venv if the base changes.
+    exit 0
+fi
+
+if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then
+    echo "No changes detected in docker/venv.Dockerfile or the Pipfiles."
+    echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False"
+elif master_commit="$(
+        get_build "refs/heads/master" \
+        | jq -re '.value[0].sourceVersion'
+    )" \
+    && git diff --quiet "${master_commit}" -- docker/base.Dockerfile
+then
+    # Though base image hasn't changed, it's still needed to build the venv.
+    echo "Can pull base image from Docker Hub; no changes made since master."
+    echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True"
 fi
-- 
cgit v1.2.3


From 931ec33623e3b0b9c1d56621a677116be15108de Mon Sep 17 00:00:00 2001
From: MarkKoz <KozlovMark@gmail.com>
Date: Sun, 30 Jun 2019 16:35:24 -0700
Subject: CI: ensure count of builds returned by the API is > 0

---
 scripts/check_dockerfiles.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'scripts/check_dockerfiles.sh')

diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh
index 015fa41..c84c61f 100755
--- a/scripts/check_dockerfiles.sh
+++ b/scripts/check_dockerfiles.sh
@@ -24,7 +24,8 @@ get_build() {
     response="$(curl -sSL "${url}")"
 
     if [[ -z "${response}" ]] \
-        || ! printf '%s' "${response}" | jq -re '.count'
+        || ! count="$(printf '%s' "${response}" | jq -re '.count')" \
+        || (( "${count}" < 1 ))
     then
         return 1
     else
-- 
cgit v1.2.3