From a2494d4275c6bbe578156601b0fda0cf203e7af7 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Thu, 9 Sep 2021 15:53:32 +0100 Subject: Init metricity using docker-compose init volume --- manage.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) (limited to 'manage.py') diff --git a/manage.py b/manage.py index 648d6635..d8258281 100755 --- a/manage.py +++ b/manage.py @@ -141,44 +141,8 @@ class SiteManager: name="pythondiscord.local:8000" ) - @staticmethod - def run_metricity_init() -> None: - """ - Initialise metricity relations and populate with some testing data. - - This is done at run time since other projects, like Python bot, - rely on the site initialising it's own db, since they do not have - access to the init.sql file to mount a docker-compose volume. - """ - import psycopg2 - from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT - - print("Initialising metricity.") - - db_url_parts = SiteManager.parse_db_url(os.environ["DATABASE_URL"]) - conn = psycopg2.connect( - host=db_url_parts.hostname, - port=db_url_parts.port, - user=db_url_parts.username, - password=db_url_parts.password, - database=db_url_parts.path[1:] - ) - # Required to create a db from `cursor.execute()` - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - with conn.cursor() as cursor, open("postgres/init.sql", encoding="utf-8") as f: - cursor.execute( - f.read(), - ("metricity", db_url_parts.username, db_url_parts.password) - ) - conn.close() - def prepare_server(self) -> None: """Perform preparation tasks before running the server.""" - self.wait_for_postgres() - if self.debug: - self.run_metricity_init() - django.setup() print("Applying migrations.") @@ -236,7 +200,6 @@ def main() -> None: # Always run metricity init when in CI, indicated by the CI env var if os.environ.get("CI", "false").lower() == "true": SiteManager.wait_for_postgres() - SiteManager.run_metricity_init() # Use the custom site manager for launching the server if len(sys.argv) > 1 and sys.argv[1] == "run": -- cgit v1.2.3 From c9aed176706852196fec0fc65ee7552d511316f2 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Thu, 9 Sep 2021 15:54:17 +0100 Subject: Move psql health check to docker compose file --- docker-compose.yml | 8 +++++++- manage.py | 52 ---------------------------------------------------- 2 files changed, 7 insertions(+), 53 deletions(-) (limited to 'manage.py') diff --git a/docker-compose.yml b/docker-compose.yml index 05867a46..eb987624 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,11 @@ services: POSTGRES_DB: pysite POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + healthcheck: + test: ["CMD-SHELL", "pg_isready -U pysite"] + interval: 2s + timeout: 1s + retries: 5 volumes: - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql @@ -35,7 +40,8 @@ services: ports: - "127.0.0.1:8000:8000" depends_on: - - postgres + postgres: + condition: service_healthy tty: true volumes: - .:/app:ro diff --git a/manage.py b/manage.py index d8258281..578f4748 100755 --- a/manage.py +++ b/manage.py @@ -1,9 +1,6 @@ #!/usr/bin/env python import os -import socket import sys -import time -from urllib.parse import SplitResult, urlsplit import django from django.contrib.auth import get_user_model @@ -54,21 +51,6 @@ class SiteManager: os.environ.setdefault("DEBUG", "true") print("Starting in debug mode.") - @staticmethod - def parse_db_url(db_url: str) -> SplitResult: - """Validate and split the given databse url.""" - db_url_parts = urlsplit(db_url) - if not all(( - db_url_parts.hostname, - db_url_parts.username, - db_url_parts.password, - db_url_parts.path - )): - raise ValueError( - "The DATABASE_URL environment variable is not a valid PostgreSQL database URL." - ) - return db_url_parts - @staticmethod def create_superuser() -> None: """Create a default django admin super user in development environments.""" @@ -98,36 +80,6 @@ class SiteManager: else: print(f"Existing bot token found: {token}") - @staticmethod - def wait_for_postgres() -> None: - """Wait for the PostgreSQL database specified in DATABASE_URL.""" - print("Waiting for PostgreSQL database.") - - # Get database URL based on environmental variable passed in compose - database_url_parts = SiteManager.parse_db_url(os.environ["DATABASE_URL"]) - domain = database_url_parts.hostname - # Port may be omitted, 5432 is the default psql port - port = database_url_parts.port or 5432 - - # Attempt to connect to the database socket - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - attempts_left = 10 - while attempts_left: - try: - # Ignore 'incomplete startup packet' - s.connect((domain, port)) - s.shutdown(socket.SHUT_RDWR) - print("Database is ready.") - break - except socket.error: - attempts_left -= 1 - print("Not ready yet, retrying.") - time.sleep(0.5) - else: - print("Database could not be found, exiting.") - sys.exit(1) - @staticmethod def set_dev_site_name() -> None: """Set the development site domain in admin from default example.""" @@ -197,10 +149,6 @@ class SiteManager: def main() -> None: """Entry point for Django management script.""" - # Always run metricity init when in CI, indicated by the CI env var - if os.environ.get("CI", "false").lower() == "true": - SiteManager.wait_for_postgres() - # Use the custom site manager for launching the server if len(sys.argv) > 1 and sys.argv[1] == "run": SiteManager(sys.argv).run_server() -- cgit v1.2.3 From e9c4cdf56e2efd65786a4cf4aee0bb4e4e56bc95 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:28:39 +0300 Subject: Adds Django Distill To Project Adds django-distill to dependencies, and lays the basic groundwork to start building static routes. Adds a poetry task to help with testing. --- manage.py | 31 +++++++++++++++++++++++++++++++ poetry.lock | 17 ++++++++++++++++- pydis_site/settings.py | 21 ++++++++++++++++----- pyproject.toml | 2 ++ 4 files changed, 65 insertions(+), 6 deletions(-) (limited to 'manage.py') diff --git a/manage.py b/manage.py index 578f4748..357134ec 100755 --- a/manage.py +++ b/manage.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import os +import platform import sys +from pathlib import Path import django from django.contrib.auth import get_user_model @@ -147,6 +149,22 @@ class SiteManager: gunicorn.app.wsgiapp.run() +def clean_up_static_files(build_folder: Path) -> None: + """Recursively loop over the build directory and fix links.""" + for file in build_folder.iterdir(): + if file.is_dir(): + clean_up_static_files(file) + elif file.name.endswith(".html"): + # Fix parent host url + new = file.read_text(encoding="utf-8").replace(f"//{os.getenv('PARENT_HOST')}", "") + + # Fix windows paths if on windows + if platform.system() == "Windows": + new = new.replace("%5C", "/") + + file.write_text(new, encoding="utf-8") + + def main() -> None: """Entry point for Django management script.""" # Use the custom site manager for launching the server @@ -155,8 +173,21 @@ def main() -> None: # Pass any others directly to standard management commands else: + if _static_build := "distill" in sys.argv[1]: + # Build a static version of the site with no databases and API support + os.environ["STATIC_BUILD"] = "True" + if not os.getenv("PARENT_HOST"): + os.environ["PARENT_HOST"] = "REPLACE_THIS.HOST" + execute_from_command_line(sys.argv) + if _static_build: + # Clean up parent host in generated files + for arg in sys.argv[2:]: + if not arg.startswith("-"): + clean_up_static_files(Path(arg)) + break + if __name__ == '__main__': main() diff --git a/poetry.lock b/poetry.lock index c6724cfc..eac58fdb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -137,6 +137,18 @@ sqlparse = ">=0.2.2" argon2 = ["argon2-cffi (>=16.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-distill" +version = "2.9.0" +description = "Static site renderer and publisher for Django." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +django = "*" +requests = "*" + [[package]] name = "django-environ" version = "0.4.5" @@ -757,7 +769,7 @@ brotli = ["brotli"] [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "ed7da8dbc905d4f2c47e01301b49c4aed0083bee269da0ee5ebcc3abee4ab1a0" +content-hash = "9f0c069c14e2dbff63d58474702693f0c02b8cfd30e5af38303975a73b71bcfd" [metadata.files] asgiref = [ @@ -858,6 +870,9 @@ django = [ {file = "Django-3.0.14-py3-none-any.whl", hash = "sha256:9bc7aa619ed878fedba62ce139abe663a147dccfd20e907725ec11e02a1ca225"}, {file = "Django-3.0.14.tar.gz", hash = "sha256:d58d8394036db75a81896037d757357e79406e8f68816c3e8a28721c1d9d4c11"}, ] +django-distill = [ + {file = "django-distill-2.9.0.tar.gz", hash = "sha256:08f31dcde2e79e73c0bc4f36941830603a811cc89472be11f79f14affb460d84"}, +] django-environ = [ {file = "django-environ-0.4.5.tar.gz", hash = "sha256:6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde"}, {file = "django_environ-0.4.5-py2.py3-none-any.whl", hash = "sha256:c57b3c11ec1f319d9474e3e5a79134f40174b17c7cc024bbb2fad84646b120c4"}, diff --git a/pydis_site/settings.py b/pydis_site/settings.py index d7b87f33..d38c298b 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -25,7 +25,8 @@ from pydis_site.constants import GIT_SHA env = environ.Env( DEBUG=(bool, False), SITE_DSN=(str, ""), - BUILDING_DOCKER=(bool, False) + BUILDING_DOCKER=(bool, False), + STATIC_BUILD=(bool, False), ) sentry_sdk.init( @@ -65,10 +66,14 @@ else: SECRET_KEY = env('SECRET_KEY') # Application definition -INSTALLED_APPS = [ +NON_STATIC_APPS = [ 'pydis_site.apps.api', - 'pydis_site.apps.home', 'pydis_site.apps.staff', +] if not env("STATIC_BUILD") else [] + +INSTALLED_APPS = [ + *NON_STATIC_APPS, + 'pydis_site.apps.home', 'pydis_site.apps.resources', 'pydis_site.apps.content', 'pydis_site.apps.events', @@ -86,14 +91,20 @@ INSTALLED_APPS = [ 'django_simple_bulma', 'rest_framework', 'rest_framework.authtoken', + + 'django_distill', ] if not env("BUILDING_DOCKER"): INSTALLED_APPS.append("django_prometheus") +NON_STATIC_MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', +] if not env("STATIC_BUILD") else [] + # Ensure that Prometheus middlewares are first and last here. MIDDLEWARE = [ - 'django_prometheus.middleware.PrometheusBeforeMiddleware', + *NON_STATIC_MIDDLEWARE, 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', @@ -134,7 +145,7 @@ WSGI_APPLICATION = 'pydis_site.wsgi.application' DATABASES = { 'default': env.db(), 'metricity': env.db('METRICITY_DB_URL'), -} +} if not env("STATIC_BUILD") else {} # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators diff --git a/pyproject.toml b/pyproject.toml index d0beb632..2f1322e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ sentry-sdk = "~=0.19" markdown = "~=3.3.4" python-frontmatter = "~=1.0" django-prometheus = "~=2.1" +django-distill = "~=2.9.0" [tool.poetry.dev-dependencies] coverage = "~=5.0" @@ -53,3 +54,4 @@ test = "coverage run manage.py test" report = "coverage report -m" lint = "pre-commit run --all-files" precommit = "pre-commit install" +static = "python mange.py distill-local build --traceback --force" -- cgit v1.2.3 From 03ad772dd15c4f8214a1b0addb3ed4f0aec666d6 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 15:34:00 +0300 Subject: Move Static Build Into Separate Variable Signed-off-by: Hassan Abouelela --- manage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'manage.py') diff --git a/manage.py b/manage.py index 357134ec..90912da3 100755 --- a/manage.py +++ b/manage.py @@ -173,7 +173,9 @@ def main() -> None: # Pass any others directly to standard management commands else: - if _static_build := "distill" in sys.argv[1]: + _static_build = "distill" in sys.argv[1] + + if _static_build: # Build a static version of the site with no databases and API support os.environ["STATIC_BUILD"] = "True" if not os.getenv("PARENT_HOST"): -- cgit v1.2.3