From 8e98a48420be718973b9a5b8aec83d4133ddc6e9 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 3 Mar 2020 08:32:57 -0800 Subject: CI: make env vars used for coverage into pipeline variables Makes the script for the coverage step cleaner. --- azure-pipelines.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 16d1b7a2a..d97a13659 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,6 +14,12 @@ jobs: variables: PIP_CACHE_DIR: ".cache/pip" PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache + BOT_API_KEY: foo + BOT_SENTRY_DSN: blah + BOT_TOKEN: bar + REDDIT_CLIENT_ID: spam + REDDIT_SECRET: ham + WOLFRAM_API_KEY: baz steps: - task: UsePythonVersion@0 @@ -50,7 +56,7 @@ jobs: - script: pre-commit run --all-files displayName: 'Run pre-commit hooks' - - script: BOT_API_KEY=foo BOT_SENTRY_DSN=blah BOT_TOKEN=bar WOLFRAM_API_KEY=baz REDDIT_CLIENT_ID=spam REDDIT_SECRET=ham coverage run -m xmlrunner + - script: coverage run -m xmlrunner displayName: Run tests - script: coverage report -m && coverage xml -o coverage.xml -- cgit v1.2.3 From 5e315135e8b33658c19de7c249adefe33f79443f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 2 Mar 2020 19:48:49 -0800 Subject: CI: cache Python dependencies Reduces frequency of using pipenv to install dependencies in CI. Works by caching the entire Python directory. Only a full cache hit will skip the pipenv steps; a partial cache hit will still be followed by using pipenv to install from the pipfiles. * Disable pip cache --- azure-pipelines.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d97a13659..d7cf03aae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,6 +1,7 @@ # https://aka.ms/yaml variables: + PIP_NO_CACHE_DIR: false PIPENV_HIDE_EMOJIS: 1 PIPENV_IGNORE_VIRTUALENVS: 1 PIPENV_NOSPIN: 1 @@ -12,7 +13,6 @@ jobs: vmImage: ubuntu-18.04 variables: - PIP_CACHE_DIR: ".cache/pip" PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache BOT_API_KEY: foo BOT_SENTRY_DSN: blah @@ -29,11 +29,24 @@ jobs: versionSpec: '3.8.x' addToPath: true + - task: Cache@2 + displayName: 'Restore Python environment' + inputs: + key: python | $(Agent.OS) | "$(PythonVersion.pythonLocation)" | ./Pipfile | ./Pipfile.lock + restoreKeys: | + python | "$(PythonVersion.pythonLocation)" | ./Pipfile.lock + python | "$(PythonVersion.pythonLocation)" | ./Pipfile + python | "$(PythonVersion.pythonLocation)" + cacheHitVar: PY_ENV_RESTORED + path: $(PythonVersion.pythonLocation) + - script: pip install pipenv displayName: 'Install pipenv' + condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) - script: pipenv install --dev --deploy --system displayName: 'Install project using pipenv' + condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) # Create an executable shell script which replaces the original pipenv binary. # The shell script ignores the first argument and executes the rest of the args as a command. -- cgit v1.2.3 From 952e46350bfd95521713f36fa0afcd4eb613026a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 9 Mar 2020 17:33:17 -0700 Subject: CI: cache the Python user base dir --- azure-pipelines.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d7cf03aae..45c6699b5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,6 +5,8 @@ variables: PIPENV_HIDE_EMOJIS: 1 PIPENV_IGNORE_VIRTUALENVS: 1 PIPENV_NOSPIN: 1 + PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache + PYTHONUSERBASE: $(Pipeline.Workspace)/py-user-base jobs: - job: test @@ -13,7 +15,6 @@ jobs: vmImage: ubuntu-18.04 variables: - PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache BOT_API_KEY: foo BOT_SENTRY_DSN: blah BOT_TOKEN: bar @@ -38,13 +39,14 @@ jobs: python | "$(PythonVersion.pythonLocation)" | ./Pipfile python | "$(PythonVersion.pythonLocation)" cacheHitVar: PY_ENV_RESTORED - path: $(PythonVersion.pythonLocation) + path: $(PYTHONUSERBASE) - script: pip install pipenv displayName: 'Install pipenv' condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) - - script: pipenv install --dev --deploy --system + # PIP_USER=1 will install packages to the user site. + - script: export PIP_USER=1; pipenv install --dev --deploy --system displayName: 'Install project using pipenv' condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) -- cgit v1.2.3 From e9d58e4e021d840a569c382a5494faa52e2ef260 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 9 Mar 2020 17:36:55 -0700 Subject: CI: prepend py user base to PATH --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 45c6699b5..079530cc8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -50,6 +50,9 @@ jobs: displayName: 'Install project using pipenv' condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) + - script: echo '##vso[task.prependpath]$(PYTHONUSERBASE)/bin' + displayName: 'Prepend PATH' + # Create an executable shell script which replaces the original pipenv binary. # The shell script ignores the first argument and executes the rest of the args as a command. # It makes the `pipenv run flake8` command in the pre-commit hook work by circumventing -- cgit v1.2.3 From 413f09fc4f200cab75ca64ec69cf82a125246dc0 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 9 Mar 2020 17:58:54 -0700 Subject: CI: invalidate caches --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 079530cc8..3b0a23064 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,11 +33,11 @@ jobs: - task: Cache@2 displayName: 'Restore Python environment' inputs: - key: python | $(Agent.OS) | "$(PythonVersion.pythonLocation)" | ./Pipfile | ./Pipfile.lock + key: python | $(Agent.OS) | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile | ./Pipfile.lock restoreKeys: | - python | "$(PythonVersion.pythonLocation)" | ./Pipfile.lock - python | "$(PythonVersion.pythonLocation)" | ./Pipfile - python | "$(PythonVersion.pythonLocation)" + python | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile.lock + python | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile + python | "$(PythonVersion.pythonLocation)" | 0 cacheHitVar: PY_ENV_RESTORED path: $(PYTHONUSERBASE) @@ -66,9 +66,9 @@ jobs: - task: Cache@2 displayName: 'Restore pre-commit environment' inputs: - key: pre-commit | "$(PythonVersion.pythonLocation)" | .pre-commit-config.yaml + key: pre-commit | "$(PythonVersion.pythonLocation)" | 0 | .pre-commit-config.yaml restoreKeys: | - pre-commit | "$(PythonVersion.pythonLocation)" + pre-commit | "$(PythonVersion.pythonLocation)" | 0 path: $(PRE_COMMIT_HOME) - script: pre-commit run --all-files -- cgit v1.2.3 From fe3236c2e450e17f98454bf38b3b1f77298c78be Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 9 Mar 2020 18:53:53 -0700 Subject: CI: install pipenv to user site Some of pipenv's dependencies overlap with dependencies in the Pipfile. When installing from the Pipfile, any dependencies already present in the global site will not be installed again to the user site, and thus will not be cached. Therefore, pipenv is installed to the user site to ensure all dependencies get cached. * Move PATH prepend step before pipenv invocation --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b0a23064..9660b2621 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,6 +2,7 @@ variables: PIP_NO_CACHE_DIR: false + PIP_USER: 1 PIPENV_HIDE_EMOJIS: 1 PIPENV_IGNORE_VIRTUALENVS: 1 PIPENV_NOSPIN: 1 @@ -41,18 +42,17 @@ jobs: cacheHitVar: PY_ENV_RESTORED path: $(PYTHONUSERBASE) + - script: echo '##vso[task.prependpath]$(PYTHONUSERBASE)/bin' + displayName: 'Prepend PATH' + - script: pip install pipenv displayName: 'Install pipenv' condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) - # PIP_USER=1 will install packages to the user site. - - script: export PIP_USER=1; pipenv install --dev --deploy --system + - script: pipenv install --dev --deploy --system displayName: 'Install project using pipenv' condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) - - script: echo '##vso[task.prependpath]$(PYTHONUSERBASE)/bin' - displayName: 'Prepend PATH' - # Create an executable shell script which replaces the original pipenv binary. # The shell script ignores the first argument and executes the rest of the args as a command. # It makes the `pipenv run flake8` command in the pre-commit hook work by circumventing -- cgit v1.2.3 From f6ed0362300227c35477f4a01263b496464f9d2d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 9 Mar 2020 23:04:30 -0700 Subject: CI: don't do a user install for pre-commit venv Prevents the following error: Can not perform a '--user' install. User site-packages are not visible in this virtualenv. --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9660b2621..3557410c6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,7 +71,8 @@ jobs: pre-commit | "$(PythonVersion.pythonLocation)" | 0 path: $(PRE_COMMIT_HOME) - - script: pre-commit run --all-files + # pre-commit's venv doesn't allow user installs - not that they're really needed anyway. + - script: export PIP_USER=0; pre-commit run --all-files displayName: 'Run pre-commit hooks' - script: coverage run -m xmlrunner -- cgit v1.2.3 From c392ff64ac4c64ef4baa9834e6e0046e415ce0d9 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 11 Mar 2020 10:57:38 -0700 Subject: CI: rename UsePythonVersion task "python" is a shorter and clearer name. --- azure-pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3557410c6..16e4489c0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: steps: - task: UsePythonVersion@0 displayName: 'Set Python version' - name: PythonVersion + name: python inputs: versionSpec: '3.8.x' addToPath: true @@ -34,11 +34,11 @@ jobs: - task: Cache@2 displayName: 'Restore Python environment' inputs: - key: python | $(Agent.OS) | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile | ./Pipfile.lock + key: python | $(Agent.OS) | "$(python.pythonLocation)" | 0 | ./Pipfile | ./Pipfile.lock restoreKeys: | - python | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile.lock - python | "$(PythonVersion.pythonLocation)" | 0 | ./Pipfile - python | "$(PythonVersion.pythonLocation)" | 0 + python | "$(python.pythonLocation)" | 0 | ./Pipfile.lock + python | "$(python.pythonLocation)" | 0 | ./Pipfile + python | "$(python.pythonLocation)" | 0 cacheHitVar: PY_ENV_RESTORED path: $(PYTHONUSERBASE) @@ -59,16 +59,16 @@ jobs: # pipenv entirely, which is too dumb to know it should use the system interpreter rather than # creating a new venv. - script: | - printf '%s\n%s' '#!/bin/bash' '"${@:2}"' > $(PythonVersion.pythonLocation)/bin/pipenv \ - && chmod +x $(PythonVersion.pythonLocation)/bin/pipenv + printf '%s\n%s' '#!/bin/bash' '"${@:2}"' > $(python.pythonLocation)/bin/pipenv \ + && chmod +x $(python.pythonLocation)/bin/pipenv displayName: 'Mock pipenv binary' - task: Cache@2 displayName: 'Restore pre-commit environment' inputs: - key: pre-commit | "$(PythonVersion.pythonLocation)" | 0 | .pre-commit-config.yaml + key: pre-commit | "$(python.pythonLocation)" | 0 | .pre-commit-config.yaml restoreKeys: | - pre-commit | "$(PythonVersion.pythonLocation)" | 0 + pre-commit | "$(python.pythonLocation)" | 0 path: $(PRE_COMMIT_HOME) # pre-commit's venv doesn't allow user installs - not that they're really needed anyway. -- cgit v1.2.3 From 95db4c912787921cf8324e9f65ad4ee1bbb898bd Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 25 Mar 2020 10:11:28 -0700 Subject: CI: remove support for partial cache hits Partial hits may cause issues when packages get removed. The cache will get bloated with packages which are no longer needed. They will keep accumulating as more packages get removed unless the cache is unused for 7 days and gets automatically deleted by Azure Pipelines. Lingering packages are also a potential cause for conflicts (e.g. unused package x depends on package y==4.0 and useful package z depends on y==5.0). Removing support for partial hits means all dependencies will be installed whenever a single dependency changes. --- azure-pipelines.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 16e4489c0..d56675029 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,10 +35,6 @@ jobs: displayName: 'Restore Python environment' inputs: key: python | $(Agent.OS) | "$(python.pythonLocation)" | 0 | ./Pipfile | ./Pipfile.lock - restoreKeys: | - python | "$(python.pythonLocation)" | 0 | ./Pipfile.lock - python | "$(python.pythonLocation)" | 0 | ./Pipfile - python | "$(python.pythonLocation)" | 0 cacheHitVar: PY_ENV_RESTORED path: $(PYTHONUSERBASE) @@ -67,8 +63,6 @@ jobs: displayName: 'Restore pre-commit environment' inputs: key: pre-commit | "$(python.pythonLocation)" | 0 | .pre-commit-config.yaml - restoreKeys: | - pre-commit | "$(python.pythonLocation)" | 0 path: $(PRE_COMMIT_HOME) # pre-commit's venv doesn't allow user installs - not that they're really needed anyway. -- cgit v1.2.3