From e9c4cdf56e2efd65786a4cf4aee0bb4e4e56bc95 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:28:39 +0300 Subject: Adds Django Distill To Project Adds django-distill to dependencies, and lays the basic groundwork to start building static routes. Adds a poetry task to help with testing. --- manage.py | 31 +++++++++++++++++++++++++++++++ poetry.lock | 17 ++++++++++++++++- pydis_site/settings.py | 21 ++++++++++++++++----- pyproject.toml | 2 ++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/manage.py b/manage.py index 578f4748..357134ec 100755 --- a/manage.py +++ b/manage.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import os +import platform import sys +from pathlib import Path import django from django.contrib.auth import get_user_model @@ -147,6 +149,22 @@ class SiteManager: gunicorn.app.wsgiapp.run() +def clean_up_static_files(build_folder: Path) -> None: + """Recursively loop over the build directory and fix links.""" + for file in build_folder.iterdir(): + if file.is_dir(): + clean_up_static_files(file) + elif file.name.endswith(".html"): + # Fix parent host url + new = file.read_text(encoding="utf-8").replace(f"//{os.getenv('PARENT_HOST')}", "") + + # Fix windows paths if on windows + if platform.system() == "Windows": + new = new.replace("%5C", "/") + + file.write_text(new, encoding="utf-8") + + def main() -> None: """Entry point for Django management script.""" # Use the custom site manager for launching the server @@ -155,8 +173,21 @@ def main() -> None: # Pass any others directly to standard management commands else: + if _static_build := "distill" in sys.argv[1]: + # Build a static version of the site with no databases and API support + os.environ["STATIC_BUILD"] = "True" + if not os.getenv("PARENT_HOST"): + os.environ["PARENT_HOST"] = "REPLACE_THIS.HOST" + execute_from_command_line(sys.argv) + if _static_build: + # Clean up parent host in generated files + for arg in sys.argv[2:]: + if not arg.startswith("-"): + clean_up_static_files(Path(arg)) + break + if __name__ == '__main__': main() diff --git a/poetry.lock b/poetry.lock index c6724cfc..eac58fdb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -137,6 +137,18 @@ sqlparse = ">=0.2.2" argon2 = ["argon2-cffi (>=16.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-distill" +version = "2.9.0" +description = "Static site renderer and publisher for Django." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +django = "*" +requests = "*" + [[package]] name = "django-environ" version = "0.4.5" @@ -757,7 +769,7 @@ brotli = ["brotli"] [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "ed7da8dbc905d4f2c47e01301b49c4aed0083bee269da0ee5ebcc3abee4ab1a0" +content-hash = "9f0c069c14e2dbff63d58474702693f0c02b8cfd30e5af38303975a73b71bcfd" [metadata.files] asgiref = [ @@ -858,6 +870,9 @@ django = [ {file = "Django-3.0.14-py3-none-any.whl", hash = "sha256:9bc7aa619ed878fedba62ce139abe663a147dccfd20e907725ec11e02a1ca225"}, {file = "Django-3.0.14.tar.gz", hash = "sha256:d58d8394036db75a81896037d757357e79406e8f68816c3e8a28721c1d9d4c11"}, ] +django-distill = [ + {file = "django-distill-2.9.0.tar.gz", hash = "sha256:08f31dcde2e79e73c0bc4f36941830603a811cc89472be11f79f14affb460d84"}, +] django-environ = [ {file = "django-environ-0.4.5.tar.gz", hash = "sha256:6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde"}, {file = "django_environ-0.4.5-py2.py3-none-any.whl", hash = "sha256:c57b3c11ec1f319d9474e3e5a79134f40174b17c7cc024bbb2fad84646b120c4"}, diff --git a/pydis_site/settings.py b/pydis_site/settings.py index d7b87f33..d38c298b 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -25,7 +25,8 @@ from pydis_site.constants import GIT_SHA env = environ.Env( DEBUG=(bool, False), SITE_DSN=(str, ""), - BUILDING_DOCKER=(bool, False) + BUILDING_DOCKER=(bool, False), + STATIC_BUILD=(bool, False), ) sentry_sdk.init( @@ -65,10 +66,14 @@ else: SECRET_KEY = env('SECRET_KEY') # Application definition -INSTALLED_APPS = [ +NON_STATIC_APPS = [ 'pydis_site.apps.api', - 'pydis_site.apps.home', 'pydis_site.apps.staff', +] if not env("STATIC_BUILD") else [] + +INSTALLED_APPS = [ + *NON_STATIC_APPS, + 'pydis_site.apps.home', 'pydis_site.apps.resources', 'pydis_site.apps.content', 'pydis_site.apps.events', @@ -86,14 +91,20 @@ INSTALLED_APPS = [ 'django_simple_bulma', 'rest_framework', 'rest_framework.authtoken', + + 'django_distill', ] if not env("BUILDING_DOCKER"): INSTALLED_APPS.append("django_prometheus") +NON_STATIC_MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', +] if not env("STATIC_BUILD") else [] + # Ensure that Prometheus middlewares are first and last here. MIDDLEWARE = [ - 'django_prometheus.middleware.PrometheusBeforeMiddleware', + *NON_STATIC_MIDDLEWARE, 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', @@ -134,7 +145,7 @@ WSGI_APPLICATION = 'pydis_site.wsgi.application' DATABASES = { 'default': env.db(), 'metricity': env.db('METRICITY_DB_URL'), -} +} if not env("STATIC_BUILD") else {} # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators diff --git a/pyproject.toml b/pyproject.toml index d0beb632..2f1322e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ sentry-sdk = "~=0.19" markdown = "~=3.3.4" python-frontmatter = "~=1.0" django-prometheus = "~=2.1" +django-distill = "~=2.9.0" [tool.poetry.dev-dependencies] coverage = "~=5.0" @@ -53,3 +54,4 @@ test = "coverage run manage.py test" report = "coverage report -m" lint = "pre-commit run --all-files" precommit = "pre-commit install" +static = "python mange.py distill-local build --traceback --force" -- cgit v1.2.3 From 247c4ba220f9ee73767f4aac2737e520e7ec4b07 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:30:06 +0300 Subject: Adds Static Route Configurations Adds configuration which specifies how routes should be handled when building a static preview. --- pydis_site/apps/content/urls.py | 43 +++++++++++++++++++++++++++++++++++--- pydis_site/apps/events/urls.py | 35 ++++++++++++++++++++++++++++--- pydis_site/apps/home/urls.py | 6 +++--- pydis_site/apps/home/views/home.py | 26 ++++++++++++++++------- pydis_site/apps/resources/urls.py | 22 ++++++++++++++++--- pydis_site/urls.py | 11 ++++++++-- 6 files changed, 122 insertions(+), 21 deletions(-) diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index c11b222a..fe7c2852 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -1,9 +1,46 @@ -from django.urls import path +import typing +from pathlib import Path + +from django_distill import distill_path from . import views app_name = "content" + + +def __get_all_files(root: Path, folder: typing.Optional[Path] = None) -> list[str]: + """Find all folders and markdown files recursively starting from `root`.""" + if not folder: + folder = root + + results = [] + + for item in folder.iterdir(): + name = item.relative_to(root).__str__().replace("\\", "/") + + if item.is_dir(): + results.append(name) + results.extend(__get_all_files(root, item)) + else: + path, extension = name.rsplit(".", maxsplit=1) + if extension == "md": + results.append(path) + + return results + + +def get_all_pages() -> typing.Iterator[dict[str, str]]: + """Yield a dict of all pag categories.""" + for location in __get_all_files(Path("pydis_site", "apps", "content", "resources")): + yield {"location": location} + + urlpatterns = [ - path("", views.PageOrCategoryView.as_view(), name='pages'), - path("/", views.PageOrCategoryView.as_view(), name='page_category'), + distill_path("", views.PageOrCategoryView.as_view(), name='pages'), + distill_path( + "/", + views.PageOrCategoryView.as_view(), + name='page_category', + distill_func=get_all_pages + ), ] diff --git a/pydis_site/apps/events/urls.py b/pydis_site/apps/events/urls.py index 9a65cf1f..7ea65a31 100644 --- a/pydis_site/apps/events/urls.py +++ b/pydis_site/apps/events/urls.py @@ -1,9 +1,38 @@ -from django.urls import path +import typing +from pathlib import Path + +from django_distill import distill_path from pydis_site.apps.events.views import IndexView, PageView app_name = "events" + + +def __get_all_files(root: Path, folder: typing.Optional[Path] = None) -> list[str]: + """Find all folders and HTML files recursively starting from `root`.""" + if not folder: + folder = root + + results = [] + + for sub_folder in folder.iterdir(): + results.append( + sub_folder.relative_to(root).__str__().replace("\\", "/").replace(".html", "") + ) + + if sub_folder.is_dir(): + results.extend(__get_all_files(root, sub_folder)) + + return results + + +def get_all_events() -> typing.Iterator[dict[str, str]]: + """Yield a dict of all event pages.""" + for file in __get_all_files(Path("pydis_site", "templates", "events", "pages")): + yield {"path": file} + + urlpatterns = [ - path("", IndexView.as_view(), name="index"), - path("/", PageView.as_view(), name="page"), + distill_path("", IndexView.as_view(), name="index"), + distill_path("/", PageView.as_view(), name="page", distill_func=get_all_events), ] diff --git a/pydis_site/apps/home/urls.py b/pydis_site/apps/home/urls.py index 57abc942..30321ece 100644 --- a/pydis_site/apps/home/urls.py +++ b/pydis_site/apps/home/urls.py @@ -1,9 +1,9 @@ -from django.urls import path +from django_distill import distill_path from .views import HomeView, timeline app_name = 'home' urlpatterns = [ - path('', HomeView.as_view(), name='home'), - path('timeline/', timeline, name="timeline"), + distill_path('', HomeView.as_view(), name='home'), + distill_path('timeline/', timeline, name="timeline"), ] diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 401c768f..e28a3a00 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -8,6 +8,7 @@ from django.shortcuts import render from django.utils import timezone from django.views import View +from pydis_site import settings from pydis_site.apps.home.models import RepositoryMetadata from pydis_site.constants import GITHUB_TOKEN, TIMEOUT_PERIOD @@ -32,7 +33,10 @@ class HomeView(View): def __init__(self): """Clean up stale RepositoryMetadata.""" - RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete() + self._static_build = settings.env("STATIC_BUILD") + + if not self._static_build: + RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete() # If no token is defined (for example in local development), then # it does not make sense to pass the Authorization header. More @@ -91,10 +95,13 @@ class HomeView(View): def _get_repo_data(self) -> List[RepositoryMetadata]: """Build a list of RepositoryMetadata objects that we can use to populate the front page.""" # First off, load the timestamp of the least recently updated entry. - last_update = ( - RepositoryMetadata.objects.values_list("last_updated", flat=True) - .order_by("last_updated").first() - ) + if self._static_build: + last_update = None + else: + last_update = ( + RepositoryMetadata.objects.values_list("last_updated", flat=True) + .order_by("last_updated").first() + ) # If we did not retrieve any results here, we should import them! if last_update is None: @@ -104,7 +111,7 @@ class HomeView(View): api_repositories = self._get_api_data() # Create all the repodata records in the database. - return RepositoryMetadata.objects.bulk_create( + data = [ RepositoryMetadata( repo_name=api_data["full_name"], description=api_data["description"], @@ -113,7 +120,12 @@ class HomeView(View): language=api_data["language"], ) for api_data in api_repositories.values() - ) + ] + + if settings.env("STATIC_BUILD"): + return data + else: + return RepositoryMetadata.objects.bulk_create(data) # If the data is stale, we should refresh it. if (timezone.now() - last_update).seconds > self.repository_cache_ttl: diff --git a/pydis_site/apps/resources/urls.py b/pydis_site/apps/resources/urls.py index 19142081..10eda132 100644 --- a/pydis_site/apps/resources/urls.py +++ b/pydis_site/apps/resources/urls.py @@ -1,9 +1,25 @@ -from django.urls import path +import typing +from pathlib import Path + +from django_distill import distill_path from pydis_site.apps.resources import views app_name = "resources" + + +def get_all_resources() -> typing.Iterator[dict[str, str]]: + """Yield a dict of all resource categories.""" + for category in Path("pydis_site", "apps", "resources", "resources").iterdir(): + yield {"category": category.name} + + urlpatterns = [ - path("", views.ResourcesView.as_view(), name="index"), - path("/", views.ResourcesListView.as_view(), name="resources") + distill_path("", views.ResourcesView.as_view(), name="index"), + distill_path( + "/", + views.ResourcesListView.as_view(), + name="resources", + distill_func=get_all_resources + ), ] diff --git a/pydis_site/urls.py b/pydis_site/urls.py index 891dbdcc..51ef4214 100644 --- a/pydis_site/urls.py +++ b/pydis_site/urls.py @@ -1,8 +1,9 @@ from django.contrib import admin from django.urls import include, path +from pydis_site import settings -urlpatterns = ( +NON_STATIC_PATTERNS = [ path('admin/', admin.site.urls), # External API ingress (over the net) @@ -14,9 +15,15 @@ urlpatterns = ( # from wildcard matching all requests to `pages/...`. path('', include('pydis_site.apps.redirect.urls')), path('', include('django_prometheus.urls')), + + path('staff/', include('pydis_site.apps.staff.urls', namespace='staff')), +] if not settings.env("STATIC_BUILD") else [] + + +urlpatterns = ( + *NON_STATIC_PATTERNS, path('pages/', include('pydis_site.apps.content.urls', namespace='content')), path('resources/', include('pydis_site.apps.resources.urls')), path('events/', include('pydis_site.apps.events.urls', namespace='events')), - path('staff/', include('pydis_site.apps.staff.urls', namespace='staff')), path('', include('pydis_site.apps.home.urls', namespace='home')), ) -- cgit v1.2.3 From 595185d826286b7b832d66a5b35e0f7ecbe29c55 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:30:55 +0300 Subject: Adds Static Docker Build Adds a docker image which performs a static build of the site. --- static-builds/Dockerfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 static-builds/Dockerfile diff --git a/static-builds/Dockerfile b/static-builds/Dockerfile new file mode 100644 index 00000000..a70165a0 --- /dev/null +++ b/static-builds/Dockerfile @@ -0,0 +1,28 @@ +# Build a static preview of the site +FROM --platform=linux/amd64 python:3.9-slim-buster + +# Allow service to handle stops gracefully +STOPSIGNAL SIGQUIT + +# Set pip to have cleaner logs and no saved cache +ENV PIP_NO_CACHE_DIR=false \ + POETRY_VIRTUALENVS_CREATE=false + +# Install poetry +RUN pip install -U poetry + +# Copy the project files into working directory +WORKDIR /app + +# Install project dependencies +COPY pyproject.toml poetry.lock ./ +RUN poetry install --no-dev + +ARG git_sha="development" +ENV GIT_SHA=$git_sha +ENV PARENT_HOST=replace_me.host + +# Copy the source code in last to optimize rebuilding the image +COPY . . + +RUN SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force --collectstatic -- cgit v1.2.3 From f9ba4a8bca526cb66b513622b987f775aa88403e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:31:36 +0300 Subject: Adds Netlify Builds Adds an action which builds and uploads the static site as an artifact, and a fetch script to be run on the netlify builders. --- .github/workflows/static-preview.yaml | 69 +++++++++++++++++++ .gitignore | 3 + static-builds/README.md | 47 +++++++++++++ static-builds/netlify_build.py | 123 ++++++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 .github/workflows/static-preview.yaml create mode 100644 static-builds/README.md create mode 100644 static-builds/netlify_build.py diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml new file mode 100644 index 00000000..50deed6d --- /dev/null +++ b/.github/workflows/static-preview.yaml @@ -0,0 +1,69 @@ +name: Build & Publish Static Preview + +on: + push: + branches: + - main + pull_request: + +jobs: + build: + name: Build Static Preview + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + # Create a commit SHA-based tag for the container repositories + - name: Create SHA Container Tag + id: sha_tag + run: | + tag=$(cut -c 1-7 <<< $GITHUB_SHA) + echo "::set-output name=tag::$tag" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Build the container, including an inline cache manifest to + # allow us to use the registry as a cache source. + - name: Build & Push Base Image + uses: docker/build-push-action@v2 + if: github.ref == 'refs/heads/main' + with: + context: . + file: ./static-builds/Dockerfile + push: true + cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest + cache-to: type=inline + tags: | + ghcr.io/python-discord/static-site:latest + ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} + build-args: | + git_sha=${{ github.sha }} + + - name: Build Local Docker Image + run: | + docker build \ + --build-arg git_sha=${{ github.sha }} \ + --cache-from ghcr.io/python-discord/static-site:latest \ + -t static-site:${{ steps.sha_tag.outputs.tag }} \ + -f static-builds/Dockerfile \ + . + + - name: Extract Build From Docker Image + run: | + docker run --name site static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker cp site:/app/build build/ + + - name: Upload Build + uses: actions/upload-artifact@v2 + with: + name: static-build + path: build/ diff --git a/.gitignore b/.gitignore index e4ad2c19..08d257bc 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ staticfiles/ *.js.tmp log.* + +# Local Netlify folder +.netlify diff --git a/static-builds/README.md b/static-builds/README.md new file mode 100644 index 00000000..ee2c0910 --- /dev/null +++ b/static-builds/README.md @@ -0,0 +1,47 @@ +# Static Builds +This directory includes all the needed information to build and deploy static previews of the site. + +Static deployments use [django-distill](https://github.com/meeb/django-distill) to build the static content. +The content is built in GitHub Actions, and is fetched and deployed by Netlify. + + +## Instructions +These are the configuration instructions to get started with static deployments. +They are split into two parts: + +- [Building The Site](#building-the-site) +- [Deploying To Netlify](#deploying-to-netlify) + + +### Building The Site +To get started with building, you can use the following command: + +```shell +python -m pip install httpx==0.19.0 +python manage.py distill-local build --traceback --force --collectstatic +``` + +Alternatively, you can use the [Dockerfile](./Dockerfile) in this folder. + +Both output their builds to a `build/` directory. + +> Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. + + +### Deploying To Netlify +To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: + +Build Command: +`python -m pip install httpx==0.19.0 && python static-builds/netlify_build.py` + +Publish Directory: +`build` + +Environment Variables: +- PYTHON_VERSION: 3.8 +- TOKEN: A GitHub token with access to download build artifacts. + + +Note that at this time, if you are deploying to netlify yourself, you won't have access to the +fa-icons pack we are using, which will lead to many missing icons on your preview. +You can either update the pack to one which will work on your domain, or you'll have to live with the missing icons. diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py new file mode 100644 index 00000000..6686e2ab --- /dev/null +++ b/static-builds/netlify_build.py @@ -0,0 +1,123 @@ +"""Build script to deploy project on netlify.""" + +# WARNING: This file must remain compatible with python 3.8 + +# This script performs all the actions required to build and deploy our project on netlify +# It requires the following environment variable: + +# TOKEN: A GitHub access token that can download the artifact. +# For PAT, the only scope needed is `public_repos` + +# It depends on the following packages, which are set in the netlify UI: +# httpx == 0.19.0 + +import os +import time +import zipfile +from pathlib import Path +from urllib import parse + +import httpx + +API_URL = "https://api.github.com" +OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] + + +def get_build_artifact() -> str: + """Search for a build artifact, and return the download URL.""" + print("Fetching build URL.") + + if os.getenv("PULL_REQUEST").lower() == "true": + print(f"Fetching data for PR #{os.getenv('REVIEW_ID')}") + + pull_url = f"{API_URL}/repos/{OWNER}/{REPO}/pulls/{os.getenv('REVIEW_ID')}" + pull_request = httpx.get(pull_url) + pull_request.raise_for_status() + + commit_sha = pull_request.json()["head"]["sha"] + + workflows_params = parse.urlencode({ + "event": "pull_request", + "per_page": 100 + }) + + else: + commit_sha = os.getenv("COMMIT_REF") + + workflows_params = parse.urlencode({ + "event": "push", + "per_page": 100 + }) + + print(f"Fetching action data for commit {commit_sha}") + + workflows = httpx.get(f"{API_URL}/repos/{OWNER}/{REPO}/actions/runs?{workflows_params}") + workflows.raise_for_status() + + for run in workflows.json()["workflow_runs"]: + if run["name"] == "Build & Publish Static Preview" and commit_sha == run["head_sha"]: + print(f"Found action for this commit: {run['id']}\n{run['html_url']}") + break + else: + raise Exception("Could not find the workflow run for this event.") + + polls = 0 + while polls <= 20: + if run["status"] != "completed": + print("Action isn't completed, sleeping for 30 seconds.") + polls += 1 + time.sleep(30) + + elif run["conclusion"] != "success": + print("Aborting build due to a failure in a previous CI step.") + exit(0) + + else: + print(f"Found artifact URL:\n{run['artifacts_url']}") + return run["artifacts_url"] + + _run = httpx.get(run["url"]) + _run.raise_for_status() + run = _run.json() + + raise Exception("Polled for the artifact workflow, but it was not ready in time.") + + +def download_artifact(url: str) -> None: + """Download a build artifact from `url`, and unzip the content.""" + print("Fetching artifact data.") + + artifacts = httpx.get(url) + artifacts.raise_for_status() + artifacts = artifacts.json() + + if artifacts["total_count"] == "0": + raise Exception(f"No artifacts were found for this build, aborting.\n{url}") + + for artifact in artifacts["artifacts"]: + if artifact["name"] == "static-build": + print("Found artifact with build.") + break + else: + raise Exception("Could not find an artifact with the expected name.") + + zipped_content = httpx.get(artifact["archive_download_url"], headers={ + "Authorization": f"token {os.getenv('TOKEN')}" + }) + zipped_content.raise_for_status() + + zip_file = Path("temp.zip") + zip_file.write_bytes(zipped_content.read()) + + with zipfile.ZipFile(zip_file, "r") as zip_ref: + zip_ref.extractall("build") + + zip_file.unlink(missing_ok=True) + + print("Wrote artifact content to target directory.") + + +if __name__ == "__main__": + print("Build started") + artifact_url = get_build_artifact() + download_artifact(artifact_url) -- cgit v1.2.3 From cf199b2b84773d568d7f548fdbf6ba12f63171c3 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 13:23:02 +0300 Subject: Fixes Caching On Docker Build Reworks the docker build action to use buildx in all steps to make the caching work. Reduces the wait time on the fetch action. Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 32 +++++++++++++++++++------------- static-builds/netlify_build.py | 4 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 50deed6d..970fad99 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -33,7 +33,7 @@ jobs: # Build the container, including an inline cache manifest to # allow us to use the registry as a cache source. - - name: Build & Push Base Image + - name: Build Docker Image (Main) uses: docker/build-push-action@v2 if: github.ref == 'refs/heads/main' with: @@ -48,22 +48,28 @@ jobs: build-args: | git_sha=${{ github.sha }} - - name: Build Local Docker Image + - name: Extract Build From Docker Image (Main) + if: github.ref == 'refs/heads/main' run: | - docker build \ - --build-arg git_sha=${{ github.sha }} \ - --cache-from ghcr.io/python-discord/static-site:latest \ - -t static-site:${{ steps.sha_tag.outputs.tag }} \ - -f static-builds/Dockerfile \ - . + mkdir docker_build \ + && docker run --name site ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker cp site:/app docker_build/ - - name: Extract Build From Docker Image - run: | - docker run --name site static-site:${{ steps.sha_tag.outputs.tag }} \ - && docker cp site:/app/build build/ + # Build directly to a local folder + - name: Build Docker Image (PR) + uses: docker/build-push-action@v2 + if: github.ref != 'refs/heads/main' + with: + context: . + file: ./static-builds/Dockerfile + push: false + cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest + outputs: type=local,dest=docker_build/ + build-args: | + git_sha=${{ github.sha }} - name: Upload Build uses: actions/upload-artifact@v2 with: name: static-build - path: build/ + path: docker_build/app/build/ diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 6686e2ab..5699c3e4 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -64,9 +64,9 @@ def get_build_artifact() -> str: polls = 0 while polls <= 20: if run["status"] != "completed": - print("Action isn't completed, sleeping for 30 seconds.") + print("Action isn't ready, sleeping for 10 seconds.") polls += 1 - time.sleep(30) + time.sleep(10) elif run["conclusion"] != "success": print("Aborting build due to a failure in a previous CI step.") -- cgit v1.2.3 From 79259eb856ea2847a61c44dee4eb03f557a7e4f1 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 14:42:40 +0300 Subject: Merges Dockerfiles Merges the normal dockerfile with the static build one to reduce duplication. Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 7 ++++--- Dockerfile | 6 ++++++ static-builds/Dockerfile | 28 ---------------------------- static-builds/README.md | 2 +- 4 files changed, 11 insertions(+), 32 deletions(-) delete mode 100644 static-builds/Dockerfile diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 970fad99..06192f02 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -38,7 +38,6 @@ jobs: if: github.ref == 'refs/heads/main' with: context: . - file: ./static-builds/Dockerfile push: true cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest cache-to: type=inline @@ -47,12 +46,14 @@ jobs: ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} build-args: | git_sha=${{ github.sha }} + STATIC_BUILD=TRUE - name: Extract Build From Docker Image (Main) if: github.ref == 'refs/heads/main' run: | mkdir docker_build \ - && docker run --name site ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker run --entrypoint /bin/echo --name site \ + ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ && docker cp site:/app docker_build/ # Build directly to a local folder @@ -61,12 +62,12 @@ jobs: if: github.ref != 'refs/heads/main' with: context: . - file: ./static-builds/Dockerfile push: false cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest outputs: type=local,dest=docker_build/ build-args: | git_sha=${{ github.sha }} + STATIC_BUILD=TRUE - name: Upload Build uses: actions/upload-artifact@v2 diff --git a/Dockerfile b/Dockerfile index 046e7f80..2b039fab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,12 @@ RUN \ METRICITY_DB_URL=postgres://localhost \ python manage.py collectstatic --noinput --clear +# Build static files if we are doing a static build +ARG STATIC_BUILD=false +RUN if [ $STATIC_BUILD = "TRUE" ] ; \ + then SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force ; \ +fi + # Run web server through custom manager ENTRYPOINT ["python", "manage.py"] CMD ["run"] diff --git a/static-builds/Dockerfile b/static-builds/Dockerfile deleted file mode 100644 index a70165a0..00000000 --- a/static-builds/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Build a static preview of the site -FROM --platform=linux/amd64 python:3.9-slim-buster - -# Allow service to handle stops gracefully -STOPSIGNAL SIGQUIT - -# Set pip to have cleaner logs and no saved cache -ENV PIP_NO_CACHE_DIR=false \ - POETRY_VIRTUALENVS_CREATE=false - -# Install poetry -RUN pip install -U poetry - -# Copy the project files into working directory -WORKDIR /app - -# Install project dependencies -COPY pyproject.toml poetry.lock ./ -RUN poetry install --no-dev - -ARG git_sha="development" -ENV GIT_SHA=$git_sha -ENV PARENT_HOST=replace_me.host - -# Copy the source code in last to optimize rebuilding the image -COPY . . - -RUN SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force --collectstatic diff --git a/static-builds/README.md b/static-builds/README.md index ee2c0910..fe24df07 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -21,7 +21,7 @@ python -m pip install httpx==0.19.0 python manage.py distill-local build --traceback --force --collectstatic ``` -Alternatively, you can use the [Dockerfile](./Dockerfile) in this folder. +Alternatively, you can use the [Dockerfile](/Dockerfile) and extract the build. Both output their builds to a `build/` directory. -- cgit v1.2.3 From 8b58305b50348691e8ae12b17efba828a15958a2 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 14:54:14 +0300 Subject: Error Out On Missing Files In CI Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 06192f02..52d7df5a 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -74,3 +74,4 @@ jobs: with: name: static-build path: docker_build/app/build/ + if-no-files-found: error -- cgit v1.2.3 From 03ad772dd15c4f8214a1b0addb3ed4f0aec666d6 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 15:34:00 +0300 Subject: Move Static Build Into Separate Variable Signed-off-by: Hassan Abouelela --- manage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 357134ec..90912da3 100755 --- a/manage.py +++ b/manage.py @@ -173,7 +173,9 @@ def main() -> None: # Pass any others directly to standard management commands else: - if _static_build := "distill" in sys.argv[1]: + _static_build = "distill" in sys.argv[1] + + if _static_build: # Build a static version of the site with no databases and API support os.environ["STATIC_BUILD"] = "True" if not os.getenv("PARENT_HOST"): -- cgit v1.2.3 From 036f1b6544a3f7bf8fddd3c71d42eebca784ea98 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 20:06:19 +0300 Subject: Uses Nightly To Download Artifacts Signed-off-by: Hassan Abouelela --- static-builds/README.md | 3 ++- static-builds/netlify_build.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/static-builds/README.md b/static-builds/README.md index fe24df07..b5cba896 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -27,6 +27,8 @@ Both output their builds to a `build/` directory. > Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. +Note: The build script uses [nightly.link](https://github.com/oprypin/nightly.link) +to fetch the artifact with no verification. ### Deploying To Netlify To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: @@ -39,7 +41,6 @@ Publish Directory: Environment Variables: - PYTHON_VERSION: 3.8 -- TOKEN: A GitHub token with access to download build artifacts. Note that at this time, if you are deploying to netlify yourself, you won't have access to the diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 5699c3e4..4e1e6106 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -3,16 +3,12 @@ # WARNING: This file must remain compatible with python 3.8 # This script performs all the actions required to build and deploy our project on netlify -# It requires the following environment variable: - -# TOKEN: A GitHub access token that can download the artifact. -# For PAT, the only scope needed is `public_repos` - # It depends on the following packages, which are set in the netlify UI: # httpx == 0.19.0 import os import time +import typing import zipfile from pathlib import Path from urllib import parse @@ -20,11 +16,16 @@ from urllib import parse import httpx API_URL = "https://api.github.com" +NIGHTLY_URL = "https://nightly.link" OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] -def get_build_artifact() -> str: - """Search for a build artifact, and return the download URL.""" +def get_build_artifact() -> typing.Tuple[int, str]: + """ + Search for a build artifact, and return the result. + + The return is a tuple of the check suite ID, and the URL to the artifacts. + """ print("Fetching build URL.") if os.getenv("PULL_REQUEST").lower() == "true": @@ -74,7 +75,7 @@ def get_build_artifact() -> str: else: print(f"Found artifact URL:\n{run['artifacts_url']}") - return run["artifacts_url"] + return run["check_suite_id"], run["artifacts_url"] _run = httpx.get(run["url"]) _run.raise_for_status() @@ -83,7 +84,7 @@ def get_build_artifact() -> str: raise Exception("Polled for the artifact workflow, but it was not ready in time.") -def download_artifact(url: str) -> None: +def download_artifact(suite_id: int, url: str) -> None: """Download a build artifact from `url`, and unzip the content.""" print("Fetching artifact data.") @@ -101,9 +102,8 @@ def download_artifact(url: str) -> None: else: raise Exception("Could not find an artifact with the expected name.") - zipped_content = httpx.get(artifact["archive_download_url"], headers={ - "Authorization": f"token {os.getenv('TOKEN')}" - }) + artifact_url = f"{NIGHTLY_URL}/{OWNER}/{REPO}/suites/{suite_id}/artifacts/{artifact['id']}" + zipped_content = httpx.get(artifact_url) zipped_content.raise_for_status() zip_file = Path("temp.zip") @@ -119,5 +119,4 @@ def download_artifact(url: str) -> None: if __name__ == "__main__": print("Build started") - artifact_url = get_build_artifact() - download_artifact(artifact_url) + download_artifact(*get_build_artifact()) -- cgit v1.2.3