From 644c078dcc73f20d187d3300db9cdc9296b70e8b Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 19:00:29 -0700 Subject: CI: consistent indentation in yml file --- azure-pipelines.yml | 180 ++++++++++++++++++++++++++-------------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 450e005b..b052f1ae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,93 +1,93 @@ # https://aka.ms/yaml jobs: -- job: python_lint - displayName: 'Lint Job' - pool: - vmImage: ubuntu-16.04 - - variables: - PIP_CACHE_DIR: .cache/pip - - steps: - - task: UsePythonVersion@0 - displayName: 'Set Python Version' - inputs: - versionSpec: '3.7.x' - addToPath: true - - - script: python3 -m pip install pipenv && pipenv install --dev --system && python3 -m pip install flake8-formatter-junit-xml - displayName: 'Install Project Environment' - - - script: python3 -m flake8 --format junit-xml --output-file test-lint.xml - displayName: 'Run Linter' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Site-Django Lint Results' - -- job: coverage_test - displayName: 'Test Job' - dependsOn: python_lint - pool: - vmImage: ubuntu-16.04 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7.x' - architecture: x64 - - - script: | - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - sudo apt-get update -y - sudo apt-get install -y postgresql-11 - displayName: 'Install PostgreSQL' - - - script: | - echo "$USER_CREATE_COMMAND;" > pgscript.sql - echo "CREATE DATABASE pysite OWNER pysite;" >> pgscript.sql - sudo su postgres -c "psql < pgscript.sql" - env: - USER_CREATE_COMMAND: CREATE USER pysite WITH PASSWORD 'pysite' CREATEDB - displayName: 'Setup Database' - - - script: python3 -m pip install pipenv && pipenv install --dev --system - displayName: 'Install Project Environment' - - - script: | - python manage.py makemigrations --check - python manage.py migrate - coverage run --branch manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input - env: - CI: azure - DATABASE_URL: postgres://pysite:pysite@localhost/pysite - displayName: 'Run Test' - - - script: coverage report - displayName: 'Show Coverage Results' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/TEST-*.xml" - testRunTitle: 'Site-Django Test Results' - -- job: build - displayName: 'Build & Push Container' - dependsOn: coverage_test - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - steps: - - task: Docker@2 - displayName: 'Build & Push Container' - - inputs: - containerRegistry: 'DockerHubV2' - repository: 'pythondiscord/site' - command: 'buildAndPush' - Dockerfile: 'docker/app/Dockerfile' - buildContext: '.' - tags: 'latest' + - job: python_lint + displayName: 'Lint Job' + pool: + vmImage: ubuntu-16.04 + + variables: + PIP_CACHE_DIR: .cache/pip + + steps: + - task: UsePythonVersion@0 + displayName: 'Set Python Version' + inputs: + versionSpec: '3.7.x' + addToPath: true + + - script: python3 -m pip install pipenv && pipenv install --dev --system && python3 -m pip install flake8-formatter-junit-xml + displayName: 'Install Project Environment' + + - script: python3 -m flake8 --format junit-xml --output-file test-lint.xml + displayName: 'Run Linter' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Site-Django Lint Results' + + - job: coverage_test + displayName: 'Test Job' + dependsOn: python_lint + pool: + vmImage: ubuntu-16.04 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7.x' + architecture: x64 + + - script: | + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo apt-get update -y + sudo apt-get install -y postgresql-11 + displayName: 'Install PostgreSQL' + + - script: | + echo "$USER_CREATE_COMMAND;" > pgscript.sql + echo "CREATE DATABASE pysite OWNER pysite;" >> pgscript.sql + sudo su postgres -c "psql < pgscript.sql" + env: + USER_CREATE_COMMAND: CREATE USER pysite WITH PASSWORD 'pysite' CREATEDB + displayName: 'Setup Database' + + - script: python3 -m pip install pipenv && pipenv install --dev --system + displayName: 'Install Project Environment' + + - script: | + python manage.py makemigrations --check + python manage.py migrate + coverage run --branch manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input + env: + CI: azure + DATABASE_URL: postgres://pysite:pysite@localhost/pysite + displayName: 'Run Test' + + - script: coverage report + displayName: 'Show Coverage Results' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/TEST-*.xml" + testRunTitle: 'Site-Django Test Results' + + - job: build + displayName: 'Build & Push Container' + dependsOn: coverage_test + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + + steps: + - task: Docker@2 + displayName: 'Build & Push Container' + + inputs: + containerRegistry: 'DockerHubV2' + repository: 'pythondiscord/site' + command: 'buildAndPush' + Dockerfile: 'docker/app/Dockerfile' + buildContext: '.' + tags: 'latest' -- cgit v1.2.3 From f7c429f1f987bd1458df86b4e1396ef1da33893a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 19:45:03 -0700 Subject: CI: merge lint and test jobs --- azure-pipelines.yml | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b052f1ae..cdc4ce11 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,8 +1,8 @@ # https://aka.ms/yaml jobs: - - job: python_lint - displayName: 'Lint Job' + - job: test + displayName: 'Test & Lint' pool: vmImage: ubuntu-16.04 @@ -19,27 +19,9 @@ jobs: - script: python3 -m pip install pipenv && pipenv install --dev --system && python3 -m pip install flake8-formatter-junit-xml displayName: 'Install Project Environment' - - script: python3 -m flake8 --format junit-xml --output-file test-lint.xml + - script: python3 -m flake8 --format junit-xml --output-file TEST-lint.xml displayName: 'Run Linter' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Site-Django Lint Results' - - - job: coverage_test - displayName: 'Test Job' - dependsOn: python_lint - pool: - vmImage: ubuntu-16.04 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7.x' - architecture: x64 - - script: | curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' @@ -55,9 +37,6 @@ jobs: USER_CREATE_COMMAND: CREATE USER pysite WITH PASSWORD 'pysite' CREATEDB displayName: 'Setup Database' - - script: python3 -m pip install pipenv && pipenv install --dev --system - displayName: 'Install Project Environment' - - script: | python manage.py makemigrations --check python manage.py migrate @@ -77,7 +56,7 @@ jobs: - job: build displayName: 'Build & Push Container' - dependsOn: coverage_test + dependsOn: test condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) steps: -- cgit v1.2.3 From 23ada0b8c4956b86e0006bbb769d4cbc77a7d27d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 19:46:45 -0700 Subject: CI: publish test results even if a task fails --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cdc4ce11..f0b44db5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -50,6 +50,7 @@ jobs: displayName: 'Show Coverage Results' - task: PublishTestResults@2 + condition: succeededOrFailed() inputs: testResultsFiles: "**/TEST-*.xml" testRunTitle: 'Site-Django Test Results' -- cgit v1.2.3 From 221a6496908396460364e72cd167d19df01c29be Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 19:47:35 -0700 Subject: CI: minor refactoring * Rename some tasks * Simplify python commands by removing python3 -m * Break up some long lines --- azure-pipelines.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f0b44db5..64e26661 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,10 +16,13 @@ jobs: versionSpec: '3.7.x' addToPath: true - - script: python3 -m pip install pipenv && pipenv install --dev --system && python3 -m pip install flake8-formatter-junit-xml + - script: | + pip install pipenv + pipenv install --dev --system + pip install flake8-formatter-junit-xml displayName: 'Install Project Environment' - - script: python3 -m flake8 --format junit-xml --output-file TEST-lint.xml + - script: flake8 --format junit-xml --output-file TEST-lint.xml displayName: 'Run Linter' - script: | @@ -38,22 +41,27 @@ jobs: displayName: 'Setup Database' - script: | - python manage.py makemigrations --check - python manage.py migrate - coverage run --branch manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input + python3 manage.py makemigrations --check + python3 manage.py migrate + coverage run \ + --branch \ + manage.py test \ + --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner \ + --no-input env: CI: azure DATABASE_URL: postgres://pysite:pysite@localhost/pysite - displayName: 'Run Test' + displayName: 'Run Tests' - script: coverage report displayName: 'Show Coverage Results' - task: PublishTestResults@2 condition: succeededOrFailed() + displayName: 'Publish Test & Linting Results' inputs: - testResultsFiles: "**/TEST-*.xml" - testRunTitle: 'Site-Django Test Results' + testResultsFiles: '**/TEST-*.xml' + testRunTitle: 'Site Test Results' - job: build displayName: 'Build & Push Container' @@ -63,7 +71,6 @@ jobs: steps: - task: Docker@2 displayName: 'Build & Push Container' - inputs: containerRegistry: 'DockerHubV2' repository: 'pythondiscord/site' -- cgit v1.2.3 From 28856e1f13382f8a453daa111f784733d9e785ca Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 20:06:19 -0700 Subject: CI: publish XML coverage report * Show missing line numbers in report --- azure-pipelines.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 64e26661..b2aa7141 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,8 +53,8 @@ jobs: DATABASE_URL: postgres://pysite:pysite@localhost/pysite displayName: 'Run Tests' - - script: coverage report - displayName: 'Show Coverage Results' + - script: coverage report -m && coverage xml + displayName: 'Generate Coverage Reports' - task: PublishTestResults@2 condition: succeededOrFailed() @@ -63,6 +63,13 @@ jobs: testResultsFiles: '**/TEST-*.xml' testRunTitle: 'Site Test Results' + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Coverage Results' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '**/coverage.xml' + - job: build displayName: 'Build & Push Container' dependsOn: test -- cgit v1.2.3 From 7559978e7abaeb52623ca347b2fff1dc1d32a46f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 20:21:20 -0700 Subject: CI: omit migrations from coverage * Remove --branch option since it's already specified in .coveragerc --- .coveragerc | 1 + azure-pipelines.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 68f8305a..a49af74e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,6 +8,7 @@ source = pydis_site/apps/staff pydis_site/apps/wiki omit = + */migrations/* */admin.py */apps.py */urls.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b2aa7141..5a42085e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,7 +44,6 @@ jobs: python3 manage.py makemigrations --check python3 manage.py migrate coverage run \ - --branch \ manage.py test \ --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner \ --no-input -- cgit v1.2.3 From 68389677fd24db4e5fb6bce5565ea3445c2a5f7c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 20 Sep 2019 20:25:43 -0700 Subject: Run tests with coverage in pipenv script --- Pipfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 897d1b08..6da62a2b 100644 --- a/Pipfile +++ b/Pipfile @@ -40,6 +40,7 @@ python_version = "3.7" [scripts] makemigrations = "python manage.py makemigrations" django_shell = "python manage.py shell" -test = "python manage.py test" -lint = "flake8 pydis_site" +test = "coverage run manage.py test" +report = "coverage report -m" +lint = "flake8" precommit = "pre-commit install" -- cgit v1.2.3 From fa67b3ecd60ece794daa0d1a241e6cf2c7227e3d Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Sat, 21 Sep 2019 14:31:25 +1000 Subject: Change CI DB setup to use docker. --- azure-pipelines.yml | 28 +++++++++++----------------- docker-compose.yml | 4 ++-- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5a42085e..b3d4e6da 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,6 +16,15 @@ jobs: versionSpec: '3.7.x' addToPath: true + - task: DockerCompose@0 + displayName: 'Setup Database' + inputs: + action: Run a specific service + dockerComposeFile: docker-compose.yml + projectName: pydis_web + serviceName: postgres + ports: '7777:5432' + - script: | pip install pipenv pipenv install --dev --system @@ -25,21 +34,6 @@ jobs: - script: flake8 --format junit-xml --output-file TEST-lint.xml displayName: 'Run Linter' - - script: | - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - sudo apt-get update -y - sudo apt-get install -y postgresql-11 - displayName: 'Install PostgreSQL' - - - script: | - echo "$USER_CREATE_COMMAND;" > pgscript.sql - echo "CREATE DATABASE pysite OWNER pysite;" >> pgscript.sql - sudo su postgres -c "psql < pgscript.sql" - env: - USER_CREATE_COMMAND: CREATE USER pysite WITH PASSWORD 'pysite' CREATEDB - displayName: 'Setup Database' - - script: | python3 manage.py makemigrations --check python3 manage.py migrate @@ -49,7 +43,7 @@ jobs: --no-input env: CI: azure - DATABASE_URL: postgres://pysite:pysite@localhost/pysite + DATABASE_URL: postgres://pysite:pysite@localhost:7777/pysite displayName: 'Run Tests' - script: coverage report -m && coverage xml @@ -78,7 +72,7 @@ jobs: - task: Docker@2 displayName: 'Build & Push Container' inputs: - containerRegistry: 'DockerHubV2' + containerRegistry: 'DockerHub' repository: 'pythondiscord/site' command: 'buildAndPush' Dockerfile: 'docker/app/Dockerfile' diff --git a/docker-compose.yml b/docker-compose.yml index 0a8e6bcd..4ea110c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: - "127.0.0.1:7777:5432" environment: POSTGRES_DB: pysite - POSTGRES_PASSWORD: supersecretpassword + POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite web: @@ -32,7 +32,7 @@ services: - .:/app:ro - staticfiles:/var/www/static environment: - DATABASE_URL: postgres://pysite:supersecretpassword@postgres:5432/pysite + DATABASE_URL: postgres://pysite:pysite@postgres:5432/pysite DEBUG: "true" SECRET_KEY: suitable-for-development-only STATIC_ROOT: /var/www/static -- cgit v1.2.3