From 7a537d41d1e998076423ff21b8877dbafead2c98 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 2 Sep 2021 21:13:02 +0000 Subject: Install Django prometheus --- poetry.lock | 32 +++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 76c8890b..8e66cb57 100644 --- a/poetry.lock +++ b/poetry.lock @@ -157,6 +157,17 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "django-prometheus" +version = "2.1.0" +description = "Django middlewares to monitor your application with Prometheus.io." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +prometheus-client = ">=0.7" + [[package]] name = "django-simple-bulma" version = "2.2.0" @@ -455,6 +466,17 @@ pyyaml = ">=5.1" toml = "*" virtualenv = ">=20.0.8" +[[package]] +name = "prometheus-client" +version = "0.11.0" +description = "Python client for the Prometheus monitoring system." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +twisted = ["twisted"] + [[package]] name = "psutil" version = "5.8.0" @@ -706,7 +728,7 @@ brotli = ["brotli"] [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "741444c18250124e2d998506b0643fb26240d7481389a13648b02fa1794435e3" +content-hash = "cb345745569d389fcb35f72e34d65c1f9e860f110e521d2f634286bc9127ea2f" [metadata.files] appdirs = [ @@ -819,6 +841,10 @@ django-hosts = [ {file = "django-hosts-4.0.tar.gz", hash = "sha256:59a870d453f113c889a7888bae5408888870350e83e362740f382dad569c2281"}, {file = "django_hosts-4.0-py2.py3-none-any.whl", hash = "sha256:136ac225f34e7f2c007294441a38663ec2bba9637d870ad001def81bca87e390"}, ] +django-prometheus = [ + {file = "django-prometheus-2.1.0.tar.gz", hash = "sha256:dd3f8da1399140fbef5c00d1526a23d1ade286b144281c325f8e409a781643f2"}, + {file = "django_prometheus-2.1.0-py2.py3-none-any.whl", hash = "sha256:c338d6efde1ca336e90c540b5e87afe9287d7bcc82d651a778f302b0be17a933"}, +] django-simple-bulma = [ {file = "django-simple-bulma-2.2.0.tar.gz", hash = "sha256:dfc34839e050d5e4749498806ed7ee8c77794021efa54ab224a2de925c644fea"}, {file = "django_simple_bulma-2.2.0-py3-none-any.whl", hash = "sha256:38530d787b2b6a091b480f7cbb8841a3b3709733ebfa5e543ae339c3d8fece03"}, @@ -935,6 +961,10 @@ pre-commit = [ {file = "pre_commit-2.12.1-py2.py3-none-any.whl", hash = "sha256:70c5ec1f30406250b706eda35e868b87e3e4ba099af8787e3e8b4b01e84f4712"}, {file = "pre_commit-2.12.1.tar.gz", hash = "sha256:900d3c7e1bf4cf0374bb2893c24c23304952181405b4d88c9c40b72bda1bb8a9"}, ] +prometheus-client = [ + {file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"}, + {file = "prometheus_client-0.11.0.tar.gz", hash = "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86"}, +] psutil = [ {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"}, diff --git a/pyproject.toml b/pyproject.toml index 5bbf86ee..c8029d94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ gunicorn = "~=20.0.4" sentry-sdk = "~=0.19" markdown = "~=3.3.4" python-frontmatter = "~=1.0" +django-prometheus = "^2.1.0" [tool.poetry.dev-dependencies] coverage = "~=5.0" -- cgit v1.2.3 From 89d7dacf3337fed0685aad4409e86b3e137246ad Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 2 Sep 2021 21:15:31 +0000 Subject: Add Django Prometheus to installed apps and middleware --- Dockerfile | 1 + pydis_site/settings.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c07fc2e2..46f45bf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ COPY . . # Set dummy variables so collectstatic can load settings.py RUN \ + BUILDING_DOCKER=yes \ SECRET_KEY=dummy_value \ DATABASE_URL=postgres://localhost \ METRICITY_DB_URL=postgres://localhost \ diff --git a/pydis_site/settings.py b/pydis_site/settings.py index 7df7ad85..61f3b6f8 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -23,7 +23,8 @@ from pydis_site.constants import GIT_SHA env = environ.Env( DEBUG=(bool, False), - SITE_DSN=(str, "") + SITE_DSN=(str, ""), + BUILDING_DOCKER=(bool, False) ) sentry_sdk.init( @@ -84,10 +85,15 @@ INSTALLED_APPS = [ 'django_filters', 'django_simple_bulma', 'rest_framework', - 'rest_framework.authtoken' + 'rest_framework.authtoken', ] +if not env("BUILDING_DOCKER"): + INSTALLED_APPS.append("django_prometheus") + +# Ensure that Prometheus middlewares are first and last here. MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django_hosts.middleware.HostsRequestMiddleware', 'django.middleware.security.SecurityMiddleware', @@ -100,7 +106,9 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django_hosts.middleware.HostsResponseMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware' ] + ROOT_URLCONF = 'pydis_site.urls' TEMPLATES = [ @@ -131,7 +139,7 @@ WSGI_APPLICATION = 'pydis_site.wsgi.application' DATABASES = { 'default': env.db(), - 'metricity': env.db('METRICITY_DB_URL'), + 'metricity': env.db(), } # Password validation -- cgit v1.2.3 From b051c454ac853a145fa961f4409fcadd8b1d5dbc Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 2 Sep 2021 21:15:52 +0000 Subject: Add Django Prometheus to URLs --- pydis_site/apps/home/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydis_site/apps/home/urls.py b/pydis_site/apps/home/urls.py index 1e2af8f3..bb77220b 100644 --- a/pydis_site/apps/home/urls.py +++ b/pydis_site/apps/home/urls.py @@ -7,6 +7,7 @@ app_name = 'home' urlpatterns = [ path('', HomeView.as_view(), name='home'), path('', include('pydis_site.apps.redirect.urls')), + path('', include('django_prometheus.urls')), path('admin/', admin.site.urls), path('resources/', include('pydis_site.apps.resources.urls')), path('pages/', include('pydis_site.apps.content.urls')), -- cgit v1.2.3 From 4a6ea09f3849f86d8307526e23362ea93365ebef Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 2 Sep 2021 21:27:54 +0000 Subject: Revert change to Metricity DB URL Co-Authored-By: jchristgit --- pydis_site/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydis_site/settings.py b/pydis_site/settings.py index 61f3b6f8..6f49763b 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -139,7 +139,7 @@ WSGI_APPLICATION = 'pydis_site.wsgi.application' DATABASES = { 'default': env.db(), - 'metricity': env.db(), + 'metricity': env.db('METRICITY_DB_URL'), } # Password validation -- cgit v1.2.3 From 1190c68005f63ff7463dbb9e35d72dbc54f60f3d Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Thu, 2 Sep 2021 23:00:39 +0100 Subject: Follow standard for dependency versioning for django-prometheus Co-authored-by: Vivaan Verma --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c8029d94..c99f1e0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ gunicorn = "~=20.0.4" sentry-sdk = "~=0.19" markdown = "~=3.3.4" python-frontmatter = "~=1.0" -django-prometheus = "^2.1.0" +django-prometheus = "~=2.1" [tool.poetry.dev-dependencies] coverage = "~=5.0" -- cgit v1.2.3 From ec64154cf68a7600f78da6d35a7166ea425b3c6f Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Fri, 3 Sep 2021 22:31:39 +0100 Subject: Add note on BUILDING_DOCKER --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 46f45bf3..7d1dab74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,10 @@ COPY . . # Set dummy variables so collectstatic can load settings.py RUN \ + # Set BUILDING_DOCKER to anything but undefined so settings.py + # does not insert django_prometheus into the list of installed apps. + # This prevents django_prometheus from attempting to connect to the database + # when the collectstatic task is ran. BUILDING_DOCKER=yes \ SECRET_KEY=dummy_value \ DATABASE_URL=postgres://localhost \ -- cgit v1.2.3 From 700cc67a9aee27c903f3d55f72c113a643219a5d Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Fri, 3 Sep 2021 22:35:21 +0100 Subject: Add gunicorn.conf.py with Prometheus specific settings --- gunicorn.conf.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 gunicorn.conf.py diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..c60b64eb --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,5 @@ +# Code taken from https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn +from prometheus_client import multiprocess + +def child_exit(server, worker): + multiprocess.mark_process_dead(worker.pid) -- cgit v1.2.3 From 6bd72f73b761264032907530a9ab1d4c4fa0e97f Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Fri, 3 Sep 2021 22:40:38 +0100 Subject: Ignore gunicorn configuration from flake8 --- .flake8 | 2 +- gunicorn.conf.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index 6690af3e..e665771f 100644 --- a/.flake8 +++ b/.flake8 @@ -3,7 +3,7 @@ max-line-length=100 docstring-convention=all import-order-style=pycharm application_import_names=pydis_site -exclude=__pycache__, venv, .venv, **/migrations/**, .cache/ +exclude=__pycache__, venv, .venv, **/migrations/**, .cache/, gunicorn.conf.py ignore= B311,W503,E226,S311,T000 # Missing Docstrings diff --git a/gunicorn.conf.py b/gunicorn.conf.py index c60b64eb..4930ae5b 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -1,5 +1,10 @@ -# Code taken from https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn +""" +Configuration file for gunicorn. + +Code taken from https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn +""" from prometheus_client import multiprocess -def child_exit(server, worker): + +def child_exit(server, worker) -> None: multiprocess.mark_process_dead(worker.pid) -- cgit v1.2.3 From 593bd1a9a6b1cef9fa16608e19a379dcf6adb203 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Sat, 4 Sep 2021 09:02:17 +0100 Subject: Explicitly set gunicorn config argument --- manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manage.py b/manage.py index a7413ed9..66ad26f4 100755 --- a/manage.py +++ b/manage.py @@ -179,6 +179,7 @@ class SiteManager: "-w", "2", "--statsd-host", "graphite.default.svc.cluster.local:8125", "--statsd-prefix", "site", + "--config", "file:gunicorn.conf.py" ] # Run gunicorn for the production server. -- cgit v1.2.3