From c73e4df6341e739aa1f4c87e8855327b3bce744b Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 29 Jun 2019 23:49:30 -0700 Subject: CI: reformat indentation of YAML --- azure-pipelines.yml | 282 ++++++++++++++++++++++++++-------------------------- 1 file changed, 141 insertions(+), 141 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 98d64bf..15ed783 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,144 +1,144 @@ # https://aka.ms/yaml jobs: -- job: test - displayName: 'Lint & Test' - - pool: - vmImage: 'Ubuntu-16.04' - - steps: - - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . - displayName: 'Build Base Image' - - - script: | - docker build -t pythondiscord/snekbox-venv:dev -f docker/venv.Dockerfile --build-arg DEV=1 . - displayName: 'Build Development Image' - - - script: | - docker run \ - -td \ - --name snekbox_test \ - --privileged \ - --network host \ - -h pdsnk-dev \ - -e PYTHONDONTWRITEBYTECODE=1 \ - -e PIPENV_PIPFILE="/snekbox/Pipfile" \ - -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ - --entrypoint /bin/ash \ - pythondiscord/snekbox-venv:dev - displayName: 'Start Container' - - - script: | - docker exec snekbox_test /bin/ash -c \ - 'pipenv run lint --format junit-xml --output-file test-lint.xml' - displayName: 'Run Linter' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - displayName: 'Publish Lint Results' - inputs: - testResultsFiles: '**/test-lint.xml' - testRunTitle: 'Lint Results' - - - script: sudo swapoff -a - displayName: 'Disable swap memory' - - - script: | - docker exec snekbox_test /bin/ash -c \ - 'pipenv run coverage run -m xmlrunner' - displayName: 'Run Unit Tests' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '**/TEST-*.xml' - testRunTitle: 'Test Results' - - - script: | - docker exec snekbox_test /bin/ash -c \ - 'pipenv run coverage xml' - displayName: 'Generate Coverage Report' - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Coverage Results' - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '**/coverage.xml' - -- job: build - displayName: 'Build' - dependsOn: test - - variables: - BASE_CHANGED: true - VENV_CHANGED: true - - steps: - - task: Docker@1 - displayName: 'Login: Docker Hub' - - inputs: - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - command: 'login' - - - script: | - REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0" - echo "Retrieving previous build's commit using $REQUEST_URL" - RESPONSE="$(curl -sSL "${REQUEST_URL}")" - - if [[ $BUILD_REASON = "PullRequest" ]]; then - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" - if [[ -z $PREV_COMMIT ]]; then - echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch." - PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH" - fi - else - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" - fi - - if [[ -n $PREV_COMMIT ]]; then - echo "Using $PREV_COMMIT to compare diffs." - - if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then - echo "No changes detected in docker/base.Dockerfile. The base image will not be built." - echo "##vso[task.setvariable variable=BASE_CHANGED]false" - fi - - if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then - echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built." - echo "##vso[task.setvariable variable=VENV_CHANGED]false" - fi - else - echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built." - fi - displayName: 'Check Changed Files' - - - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . - displayName: 'Build Base Image' - condition: and(succeeded(), eq(variables.BASE_CHANGED, 'true')) - - - script: docker build -t pythondiscord/snekbox-venv:latest -f docker/venv.Dockerfile . - displayName: 'Build Virtual Environment Image' - condition: and(succeeded(), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) - - - script: docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile . - displayName: 'Build Final Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - script: docker push pythondiscord/snekbox-base:latest - displayName: 'Push Base Image to Dockerhub' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables.BASE_CHANGED, 'true')) - - - script: docker push pythondiscord/snekbox-venv:latest - displayName: 'Push Virtual Environment Image to Dockerhub' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) - - - script: docker push pythondiscord/snekbox:latest - displayName: 'Push Final Image to Dockerhub' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + - job: test + displayName: 'Lint & Test' + + pool: + vmImage: 'Ubuntu-16.04' + + steps: + - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . + displayName: 'Build Base Image' + + - script: | + docker build -t pythondiscord/snekbox-venv:dev -f docker/venv.Dockerfile --build-arg DEV=1 . + displayName: 'Build Development Image' + + - script: | + docker run \ + -td \ + --name snekbox_test \ + --privileged \ + --network host \ + -h pdsnk-dev \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e PIPENV_PIPFILE="/snekbox/Pipfile" \ + -e ENV="${PWD}/scripts/.profile" \ + -v "${PWD}":"${PWD}" \ + -w "${PWD}"\ + --entrypoint /bin/ash \ + pythondiscord/snekbox-venv:dev + displayName: 'Start Container' + + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run lint --format junit-xml --output-file test-lint.xml' + displayName: 'Run Linter' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: 'Publish Lint Results' + inputs: + testResultsFiles: '**/test-lint.xml' + testRunTitle: 'Lint Results' + + - script: sudo swapoff -a + displayName: 'Disable swap memory' + + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run coverage run -m xmlrunner' + displayName: 'Run Unit Tests' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '**/TEST-*.xml' + testRunTitle: 'Test Results' + + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run coverage xml' + displayName: 'Generate Coverage Report' + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Coverage Results' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '**/coverage.xml' + + - job: build + displayName: 'Build' + dependsOn: test + + variables: + BASE_CHANGED: true + VENV_CHANGED: true + + steps: + - task: Docker@1 + displayName: 'Login: Docker Hub' + + inputs: + containerregistrytype: 'Container Registry' + dockerRegistryEndpoint: 'DockerHub' + command: 'login' + + - script: | + REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0" + echo "Retrieving previous build's commit using $REQUEST_URL" + RESPONSE="$(curl -sSL "${REQUEST_URL}")" + + if [[ $BUILD_REASON = "PullRequest" ]]; then + PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" + if [[ -z $PREV_COMMIT ]]; then + echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch." + PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH" + fi + else + PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" + fi + + if [[ -n $PREV_COMMIT ]]; then + echo "Using $PREV_COMMIT to compare diffs." + + if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then + echo "No changes detected in docker/base.Dockerfile. The base image will not be built." + echo "##vso[task.setvariable variable=BASE_CHANGED]false" + fi + + if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then + echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built." + echo "##vso[task.setvariable variable=VENV_CHANGED]false" + fi + else + echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built." + fi + displayName: 'Check Changed Files' + + - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . + displayName: 'Build Base Image' + condition: and(succeeded(), eq(variables.BASE_CHANGED, 'true')) + + - script: docker build -t pythondiscord/snekbox-venv:latest -f docker/venv.Dockerfile . + displayName: 'Build Virtual Environment Image' + condition: and(succeeded(), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) + + - script: docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile . + displayName: 'Build Final Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + + - script: docker push pythondiscord/snekbox-base:latest + displayName: 'Push Base Image to Dockerhub' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables.BASE_CHANGED, 'true')) + + - script: docker push pythondiscord/snekbox-venv:latest + displayName: 'Push Virtual Environment Image to Dockerhub' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) + + - script: docker push pythondiscord/snekbox:latest + displayName: 'Push Final Image to Dockerhub' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) -- cgit v1.2.3 From ff89d99f02c07c5f2daa8e7901e107e555368177 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 11:59:56 -0700 Subject: CI: refactor steps * Use boolean literals instead of strings * Use Docker tasks for building and pushing * Move displayName and condition to the top of each step --- azure-pipelines.yml | 80 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 15ed783..3d2c9e3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,15 +5,25 @@ jobs: displayName: 'Lint & Test' pool: - vmImage: 'Ubuntu-16.04' + vmImage: 'ubuntu-16.04' steps: - - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . + - task: Docker@2 displayName: 'Build Base Image' + inputs: + command: build + repository: pythondiscord/snekbox-base + tags: latest + Dockerfile: docker/base.Dockerfile - - script: | - docker build -t pythondiscord/snekbox-venv:dev -f docker/venv.Dockerfile --build-arg DEV=1 . + - task: Docker@2 displayName: 'Build Development Image' + inputs: + command: build + repository: pythondiscord/snekbox-venv + tags: dev + Dockerfile: docker/venv.Dockerfile + arguments: --build-arg DEV=1 - script: | docker run \ @@ -37,14 +47,14 @@ jobs: displayName: 'Run Linter' - task: PublishTestResults@2 - condition: succeededOrFailed() displayName: 'Publish Lint Results' + condition: succeededOrFailed() inputs: testResultsFiles: '**/test-lint.xml' testRunTitle: 'Lint Results' - script: sudo swapoff -a - displayName: 'Disable swap memory' + displayName: 'Disable Swap Memory' - script: | docker exec snekbox_test /bin/ash -c \ @@ -52,8 +62,8 @@ jobs: displayName: 'Run Unit Tests' - task: PublishTestResults@2 - condition: succeededOrFailed() displayName: 'Publish Test Results' + condition: succeededOrFailed() inputs: testResultsFiles: '**/TEST-*.xml' testRunTitle: 'Test Results' @@ -81,7 +91,6 @@ jobs: steps: - task: Docker@1 displayName: 'Login: Docker Hub' - inputs: containerregistrytype: 'Container Registry' dockerRegistryEndpoint: 'DockerHub' @@ -119,26 +128,53 @@ jobs: fi displayName: 'Check Changed Files' - - script: docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile . + - task: Docker@2 displayName: 'Build Base Image' - condition: and(succeeded(), eq(variables.BASE_CHANGED, 'true')) + condition: and(succeeded(), eq(variables.BASE_CHANGED, True)) + inputs: + command: build + repository: pythondiscord/snekbox-base + tags: latest + Dockerfile: docker/base.Dockerfile - - script: docker build -t pythondiscord/snekbox-venv:latest -f docker/venv.Dockerfile . + - task: Docker@2 displayName: 'Build Virtual Environment Image' - condition: and(succeeded(), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) + condition: and(succeeded(), or(eq(variables.BASE_CHANGED, True), eq(variables.VENV_CHANGED, True)) + inputs: + command: build + repository: pythondiscord/snekbox-venv + tags: latest + Dockerfile: docker/venv.Dockerfile - - script: docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile . + - task: Docker@2 displayName: 'Build Final Image' condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: build + repository: pythondiscord/snekbox + tags: latest + Dockerfile: docker/Dockerfile + + - task: Docker@2 + displayName: 'Push Base Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables.BASE_CHANGED, True)) + inputs: + command: push + repository: pythondiscord/snekbox-base + tags: latest - - script: docker push pythondiscord/snekbox-base:latest - displayName: 'Push Base Image to Dockerhub' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables.BASE_CHANGED, 'true')) - - - script: docker push pythondiscord/snekbox-venv:latest - displayName: 'Push Virtual Environment Image to Dockerhub' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), or(eq(variables.BASE_CHANGED, 'true'), eq(variables.VENV_CHANGED, 'true'))) + - task: Docker@2 + displayName: 'Push Virtual Environment Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), or(eq(variables.BASE_CHANGED, True), eq(variables.VENV_CHANGED, True))) + inputs: + command: push + repository: pythondiscord/snekbox-venv + tags: latest - - script: docker push pythondiscord/snekbox:latest - displayName: 'Push Final Image to Dockerhub' + - task: Docker@2 + displayName: 'Push Final Image' condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: push + repository: pythondiscord/snekbox + tags: latest -- cgit v1.2.3 From 7cedf0c308a74bcb59f65d7d90978d3c7b43bf85 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 12:17:18 -0700 Subject: CI: use multi-line folded scalars for long conditions --- azure-pipelines.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d2c9e3..591f87f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -139,7 +139,14 @@ jobs: - task: Docker@2 displayName: 'Build Virtual Environment Image' - condition: and(succeeded(), or(eq(variables.BASE_CHANGED, True), eq(variables.VENV_CHANGED, True)) + condition: > + and( + succeeded(), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) inputs: command: build repository: pythondiscord/snekbox-venv @@ -157,7 +164,12 @@ jobs: - task: Docker@2 displayName: 'Push Base Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables.BASE_CHANGED, True)) + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + eq(variables.BASE_CHANGED, True) + ) inputs: command: push repository: pythondiscord/snekbox-base @@ -165,7 +177,15 @@ jobs: - task: Docker@2 displayName: 'Push Virtual Environment Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), or(eq(variables.BASE_CHANGED, True), eq(variables.VENV_CHANGED, True))) + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) inputs: command: push repository: pythondiscord/snekbox-venv -- cgit v1.2.3 From 96b5a70dc50ce9edce6439967ce35385ad6de22f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 12:23:28 -0700 Subject: CI: move check shell script to a separate file --- azure-pipelines.yml | 35 ++++------------------------------- scripts/check_dockerfiles.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100755 scripts/check_dockerfiles.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 591f87f..bbca0b7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -96,37 +96,10 @@ jobs: dockerRegistryEndpoint: 'DockerHub' command: 'login' - - script: | - REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0" - echo "Retrieving previous build's commit using $REQUEST_URL" - RESPONSE="$(curl -sSL "${REQUEST_URL}")" - - if [[ $BUILD_REASON = "PullRequest" ]]; then - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" - if [[ -z $PREV_COMMIT ]]; then - echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch." - PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH" - fi - else - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" - fi - - if [[ -n $PREV_COMMIT ]]; then - echo "Using $PREV_COMMIT to compare diffs." - - if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then - echo "No changes detected in docker/base.Dockerfile. The base image will not be built." - echo "##vso[task.setvariable variable=BASE_CHANGED]false" - fi - - if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then - echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built." - echo "##vso[task.setvariable variable=VENV_CHANGED]false" - fi - else - echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built." - fi - displayName: 'Check Changed Files' + - task: ShellScript@2 + displayName: 'Check If Images Need to Be Built' + inputs: + scriptPath: scripts/check_dockerfiles.sh - task: Docker@2 displayName: 'Build Base Image' diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh new file mode 100755 index 0000000..07e76f8 --- /dev/null +++ b/scripts/check_dockerfiles.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0" +echo "Retrieving previous build's commit using $REQUEST_URL" +RESPONSE="$(curl -sSL "${REQUEST_URL}")" + +if [[ $BUILD_REASON = "PullRequest" ]]; then + PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" + if [[ -z $PREV_COMMIT ]]; then + echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch." + PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH" + fi +else + PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" +fi + +if [[ -n $PREV_COMMIT ]]; then + echo "Using $PREV_COMMIT to compare diffs." + + if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then + echo "No changes detected in docker/base.Dockerfile. The base image will not be built." + echo "##vso[task.setvariable variable=BASE_CHANGED]false" + fi + + if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then + echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built." + echo "##vso[task.setvariable variable=VENV_CHANGED]false" + fi +else + echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built." +fi -- cgit v1.2.3 From c50a28524733a3ad4ee9767c1a0a1bab059489f1 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 14:38:46 -0700 Subject: CI: refactor script & pull base when possible * Move script's execution to the test job * Use output variables * Use jq instead of regex for parsing JSON responses from API * Wrap to 80 columns * Make more robust by checking for command success --- azure-pipelines.yml | 25 +++++++---- scripts/check_dockerfiles.sh | 98 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 93 insertions(+), 30 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bbca0b7..7467f3b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,8 +8,15 @@ jobs: vmImage: 'ubuntu-16.04' steps: + - task: ShellScript@2 + displayName: 'Check If Images Need to Be Built' + name: check + inputs: + scriptPath: scripts/check_dockerfiles.sh + - task: Docker@2 displayName: 'Build Base Image' + condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) inputs: command: build repository: pythondiscord/snekbox-base @@ -85,8 +92,9 @@ jobs: dependsOn: test variables: - BASE_CHANGED: true - VENV_CHANGED: true + BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ] + VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ] + BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ] steps: - task: Docker@1 @@ -96,14 +104,14 @@ jobs: dockerRegistryEndpoint: 'DockerHub' command: 'login' - - task: ShellScript@2 - displayName: 'Check If Images Need to Be Built' - inputs: - scriptPath: scripts/check_dockerfiles.sh - - task: Docker@2 displayName: 'Build Base Image' - condition: and(succeeded(), eq(variables.BASE_CHANGED, True)) + condition: > + and( + succeeded(), + ne(variables.BASE_PULL, True), + eq(variables.BASE_CHANGED, True) + ) inputs: command: build repository: pythondiscord/snekbox-base @@ -141,6 +149,7 @@ jobs: and( succeeded(), ne(variables['Build.Reason'], 'PullRequest'), + ne(variables.BASE_PULL, True), eq(variables.BASE_CHANGED, True) ) inputs: diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh index 07e76f8..015fa41 100755 --- a/scripts/check_dockerfiles.sh +++ b/scripts/check_dockerfiles.sh @@ -1,31 +1,85 @@ #!/usr/bin/env bash -REQUEST_URL="https://dev.azure.com/python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?queryOrder=finishTimeDescending&resultFilter=succeeded&\$top=1&repositoryType=${BUILD_REPOSITORY_PROVIDER}&repositoryId=${BUILD_REPOSITORY_NAME}&branchName=${BUILD_SOURCEBRANCH}&api-version=5.0" -echo "Retrieving previous build's commit using $REQUEST_URL" -RESPONSE="$(curl -sSL "${REQUEST_URL}")" - -if [[ $BUILD_REASON = "PullRequest" ]]; then - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"pr\.sourceSha"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" - if [[ -z $PREV_COMMIT ]]; then - echo "Could not retrieve the previous build's commit. Falling back to the head of the target branch." - PREV_COMMIT="origin/$SYSTEM_PULLREQUEST_TARGETBRANCH" - fi -else - PREV_COMMIT="$(echo "${RESPONSE}" | grep -Po '"sourceVersion"\s*:\s*"\K.*?[^\\](?="\s*[,}])')" -fi +set -euo pipefail +exec 3>&1 # New file descriptor to stdout + +BASE_URL="https://dev.azure.com/\ +python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\ +queryOrder=finishTimeDescending&\ +resultFilter=succeeded&\ +\$top=1&\ +repositoryType=${BUILD_REPOSITORY_PROVIDER}&\ +repositoryId=${BUILD_REPOSITORY_NAME}&\ +api-version=5.0" + +get_build() { + set -e # Poor Ubuntu LTS doesn't have Bash 4.4's inherit_errexit + + local branch="${1:?"get_build: argument 1 'branch' is unset"}" + local url="${BASE_URL}&branchName=${branch}" + + printf '%s\n' "Retrieving the latest successful build using ${url}" >&3 -if [[ -n $PREV_COMMIT ]]; then - echo "Using $PREV_COMMIT to compare diffs." + local response + response="$(curl -sSL "${url}")" - if [[ -z "$(git diff $PREV_COMMIT -- docker/base.Dockerfile)" ]]; then - echo "No changes detected in docker/base.Dockerfile. The base image will not be built." - echo "##vso[task.setvariable variable=BASE_CHANGED]false" + if [[ -z "${response}" ]] \ + || ! printf '%s' "${response}" | jq -re '.count' + then + return 1 + else + printf '%s' "${response}" fi +} - if [[ -z "$(git diff $PREV_COMMIT -- docker/venv.Dockerfile Pipfile*)" ]]; then - echo "No changes detected in docker/venv.Dockerfile or the Pipfiles. The venv image will not be built." - echo "##vso[task.setvariable variable=VENV_CHANGED]false" +# Get the previous commit +if [[ "${BUILD_REASON}" = "PullRequest" ]]; then + if ! prev_commit="$( + get_build "${BUILD_SOURCEBRANCH}" \ + | jq -re '.value[0].triggerInfo."pr.sourceSha"' + )" + then + echo \ + "Could not retrieve the previous build's commit." \ + "Falling back to the head of the target branch." + + prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}" fi +elif ! prev_commit="$( + get_build "${BUILD_SOURCEBRANCH}" \ + | jq -re '.value[0].sourceVersion' + )" +then + echo \ + "No previous build was found." \ + "Either the previous build is too old and was deleted" \ + "or the branch was empty before this build." \ + "All images will be built." + exit 0 +fi + +# Compare diffs +head="$(git rev-parse HEAD)" +printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}." + +if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then + echo "No changes detected in docker/base.Dockerfile." + echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False" else - echo "No previous commit was retrieved. Either the previous build is too old and was deleted or the branch was empty before this build. All images will be built." + # Always rebuild the venv if the base changes. + exit 0 +fi + +if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then + echo "No changes detected in docker/venv.Dockerfile or the Pipfiles." + echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False" +elif master_commit="$( + get_build "refs/heads/master" \ + | jq -re '.value[0].sourceVersion' + )" \ + && git diff --quiet "${master_commit}" -- docker/base.Dockerfile +then + # Though base image hasn't changed, it's still needed to build the venv. + echo "Can pull base image from Docker Hub; no changes made since master." + echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True" fi -- cgit v1.2.3 From acd4cb74bd13ebf74ef404b8f1daf9e33cb9a789 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 14:41:02 -0700 Subject: CI: build the base image if the venv needs to be build Fix #25 --- azure-pipelines.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7467f3b..2c92594 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,7 +110,10 @@ jobs: and( succeeded(), ne(variables.BASE_PULL, True), - eq(variables.BASE_CHANGED, True) + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) ) inputs: command: build @@ -150,7 +153,10 @@ jobs: succeeded(), ne(variables['Build.Reason'], 'PullRequest'), ne(variables.BASE_PULL, True), - eq(variables.BASE_CHANGED, True) + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) ) inputs: command: push -- cgit v1.2.3 From 01cafe54f21a9a99f14b34751a2b4bcd8cb358a5 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 14:46:32 -0700 Subject: CI: don't run the build job if nothing needs to be built Resolve #26 --- azure-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2c92594..8917306 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -89,6 +89,15 @@ jobs: - job: build displayName: 'Build' + condition: > + and( + succeeded(), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True), True), + eq(coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True), True) + ) + ) dependsOn: test variables: -- cgit v1.2.3 From 7a11085fc916ec226c143bece9f6f1bd79f70e2c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 15:57:00 -0700 Subject: CI: specify build context for Docker builds --- azure-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8917306..4ea57ee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs: repository: pythondiscord/snekbox-base tags: latest Dockerfile: docker/base.Dockerfile + buildContext: . - task: Docker@2 displayName: 'Build Development Image' @@ -30,6 +31,7 @@ jobs: repository: pythondiscord/snekbox-venv tags: dev Dockerfile: docker/venv.Dockerfile + buildContext: . arguments: --build-arg DEV=1 - script: | @@ -129,6 +131,7 @@ jobs: repository: pythondiscord/snekbox-base tags: latest Dockerfile: docker/base.Dockerfile + buildContext: . - task: Docker@2 displayName: 'Build Virtual Environment Image' @@ -145,6 +148,7 @@ jobs: repository: pythondiscord/snekbox-venv tags: latest Dockerfile: docker/venv.Dockerfile + buildContext: . - task: Docker@2 displayName: 'Build Final Image' @@ -154,6 +158,7 @@ jobs: repository: pythondiscord/snekbox tags: latest Dockerfile: docker/Dockerfile + buildContext: . - task: Docker@2 displayName: 'Push Base Image' -- cgit v1.2.3 From 931ec33623e3b0b9c1d56621a677116be15108de Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 30 Jun 2019 16:35:24 -0700 Subject: CI: ensure count of builds returned by the API is > 0 --- scripts/check_dockerfiles.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh index 015fa41..c84c61f 100755 --- a/scripts/check_dockerfiles.sh +++ b/scripts/check_dockerfiles.sh @@ -24,7 +24,8 @@ get_build() { response="$(curl -sSL "${url}")" if [[ -z "${response}" ]] \ - || ! printf '%s' "${response}" | jq -re '.count' + || ! count="$(printf '%s' "${response}" | jq -re '.count')" \ + || (( "${count}" < 1 )) then return 1 else -- cgit v1.2.3 From a88700790cfde2af463582fd978fa35406b61538 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 30 Jul 2019 14:28:34 -0700 Subject: Log into Docker Hub before building images in test job --- azure-pipelines.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ea57ee..3b7c1dc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,6 +14,13 @@ jobs: inputs: scriptPath: scripts/check_dockerfiles.sh + # Without a login the following Docker build tasks won't add image tags + - task: Docker@2 + displayName: 'Log into Docker Hub' + inputs: + command: login + containerRegistry: DockerHubV2 + - task: Docker@2 displayName: 'Build Base Image' condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) @@ -108,12 +115,11 @@ jobs: BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ] steps: - - task: Docker@1 - displayName: 'Login: Docker Hub' + - task: Docker@2 + displayName: 'Log into Docker Hub' inputs: - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - command: 'login' + command: login + containerRegistry: DockerHubV2 - task: Docker@2 displayName: 'Build Base Image' -- cgit v1.2.3 From 2f864a14a3f4d49d11801af45946dbd81e3e343f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 30 Jul 2019 15:31:19 -0700 Subject: Add comments to Azure Pipelines YAML * Replace some shorthand Docker command options with their full names for clarity --- azure-pipelines.yml | 29 ++++++++++++++++++++++++----- scripts/dev.sh | 9 +++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b7c1dc..f7b8eb7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,13 +14,16 @@ jobs: inputs: scriptPath: scripts/check_dockerfiles.sh - # Without a login the following Docker build tasks won't add image tags + # Without a login the following Docker build tasks won't add image tags. - task: Docker@2 displayName: 'Log into Docker Hub' inputs: command: login containerRegistry: DockerHubV2 + # The venv image depends on this image. Build it if it can't be pulled + # from Docker Hub, which will be the case if the base Dockerfile has had + # changes. - task: Docker@2 displayName: 'Build Base Image' condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) @@ -31,6 +34,7 @@ jobs: Dockerfile: docker/base.Dockerfile buildContext: . + # The dev image is never pushed and therefore is always built. - task: Docker@2 displayName: 'Build Development Image' inputs: @@ -41,18 +45,20 @@ jobs: buildContext: . arguments: --build-arg DEV=1 + # The linter and all tests run inside this container. - script: | docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev displayName: 'Start Container' @@ -69,6 +75,7 @@ jobs: testResultsFiles: '**/test-lint.xml' testRunTitle: 'Lint Results' + # Memory limit tests would fail if this isn't disabled. - script: sudo swapoff -a displayName: 'Disable Swap Memory' @@ -96,6 +103,9 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: '**/coverage.xml' + # When a pull request, only perform this job if images need to be built. + # It's always performed for non-PRs because the final image will always need + # to be built. - job: build displayName: 'Build' condition: > @@ -109,6 +119,7 @@ jobs: ) dependsOn: test + # coalesce() gives variables default values if they are null (i.e. unset). variables: BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ] VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ] @@ -121,6 +132,9 @@ jobs: command: login containerRegistry: DockerHubV2 + # Because this is the base image for the venv image, if the venv needs to + # be built, this base image must also be present. Build it if it has + # changed or can't be pulled from Docker Hub. - task: Docker@2 displayName: 'Build Base Image' condition: > @@ -139,6 +153,7 @@ jobs: Dockerfile: docker/base.Dockerfile buildContext: . + # Also build this image if base has changed - even if this image hasn't. - task: Docker@2 displayName: 'Build Virtual Environment Image' condition: > @@ -156,6 +171,7 @@ jobs: Dockerfile: docker/venv.Dockerfile buildContext: . + # Always build this image unless it's for a pull request. - task: Docker@2 displayName: 'Build Final Image' condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) @@ -166,6 +182,9 @@ jobs: Dockerfile: docker/Dockerfile buildContext: . + # Push images only after they've all successfully been built. + # These have the same conditions as the build tasks. However, for safety, + # a condition for not being a pull request is added. - task: Docker@2 displayName: 'Push Base Image' condition: > diff --git a/scripts/dev.sh b/scripts/dev.sh index 097690b..8f5b24f 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -34,16 +34,17 @@ fi # The volume is mounted to same the path in the container as the source # directory on the host to ensure coverage can find the source files. docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev \ >/dev/null \ -- cgit v1.2.3 From 4351b7b592d57ac5c5737594160890c9662fe6bf Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 5 Aug 2019 14:36:59 -0700 Subject: CI: output coverage report to stdout --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f7b8eb7..424b1a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -91,9 +91,10 @@ jobs: testResultsFiles: '**/TEST-*.xml' testRunTitle: 'Test Results' + # Run report too because the XML report doesn't output to stdout. - script: | docker exec snekbox_test /bin/ash -c \ - 'pipenv run coverage xml' + 'pipenv run /bin/ash -c "coverage report && coverage xml"' displayName: 'Generate Coverage Report' - task: PublishCodeCoverageResults@1 -- cgit v1.2.3