From 595185d826286b7b832d66a5b35e0f7ecbe29c55 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:30:55 +0300 Subject: Adds Static Docker Build Adds a docker image which performs a static build of the site. --- static-builds/Dockerfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 static-builds/Dockerfile (limited to 'static-builds') diff --git a/static-builds/Dockerfile b/static-builds/Dockerfile new file mode 100644 index 00000000..a70165a0 --- /dev/null +++ b/static-builds/Dockerfile @@ -0,0 +1,28 @@ +# Build a static preview of the site +FROM --platform=linux/amd64 python:3.9-slim-buster + +# Allow service to handle stops gracefully +STOPSIGNAL SIGQUIT + +# Set pip to have cleaner logs and no saved cache +ENV PIP_NO_CACHE_DIR=false \ + POETRY_VIRTUALENVS_CREATE=false + +# Install poetry +RUN pip install -U poetry + +# Copy the project files into working directory +WORKDIR /app + +# Install project dependencies +COPY pyproject.toml poetry.lock ./ +RUN poetry install --no-dev + +ARG git_sha="development" +ENV GIT_SHA=$git_sha +ENV PARENT_HOST=replace_me.host + +# Copy the source code in last to optimize rebuilding the image +COPY . . + +RUN SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force --collectstatic -- cgit v1.2.3 From f9ba4a8bca526cb66b513622b987f775aa88403e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:31:36 +0300 Subject: Adds Netlify Builds Adds an action which builds and uploads the static site as an artifact, and a fetch script to be run on the netlify builders. --- .github/workflows/static-preview.yaml | 69 +++++++++++++++++++ .gitignore | 3 + static-builds/README.md | 47 +++++++++++++ static-builds/netlify_build.py | 123 ++++++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 .github/workflows/static-preview.yaml create mode 100644 static-builds/README.md create mode 100644 static-builds/netlify_build.py (limited to 'static-builds') diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml new file mode 100644 index 00000000..50deed6d --- /dev/null +++ b/.github/workflows/static-preview.yaml @@ -0,0 +1,69 @@ +name: Build & Publish Static Preview + +on: + push: + branches: + - main + pull_request: + +jobs: + build: + name: Build Static Preview + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + # Create a commit SHA-based tag for the container repositories + - name: Create SHA Container Tag + id: sha_tag + run: | + tag=$(cut -c 1-7 <<< $GITHUB_SHA) + echo "::set-output name=tag::$tag" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Build the container, including an inline cache manifest to + # allow us to use the registry as a cache source. + - name: Build & Push Base Image + uses: docker/build-push-action@v2 + if: github.ref == 'refs/heads/main' + with: + context: . + file: ./static-builds/Dockerfile + push: true + cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest + cache-to: type=inline + tags: | + ghcr.io/python-discord/static-site:latest + ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} + build-args: | + git_sha=${{ github.sha }} + + - name: Build Local Docker Image + run: | + docker build \ + --build-arg git_sha=${{ github.sha }} \ + --cache-from ghcr.io/python-discord/static-site:latest \ + -t static-site:${{ steps.sha_tag.outputs.tag }} \ + -f static-builds/Dockerfile \ + . + + - name: Extract Build From Docker Image + run: | + docker run --name site static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker cp site:/app/build build/ + + - name: Upload Build + uses: actions/upload-artifact@v2 + with: + name: static-build + path: build/ diff --git a/.gitignore b/.gitignore index e4ad2c19..08d257bc 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ staticfiles/ *.js.tmp log.* + +# Local Netlify folder +.netlify diff --git a/static-builds/README.md b/static-builds/README.md new file mode 100644 index 00000000..ee2c0910 --- /dev/null +++ b/static-builds/README.md @@ -0,0 +1,47 @@ +# Static Builds +This directory includes all the needed information to build and deploy static previews of the site. + +Static deployments use [django-distill](https://github.com/meeb/django-distill) to build the static content. +The content is built in GitHub Actions, and is fetched and deployed by Netlify. + + +## Instructions +These are the configuration instructions to get started with static deployments. +They are split into two parts: + +- [Building The Site](#building-the-site) +- [Deploying To Netlify](#deploying-to-netlify) + + +### Building The Site +To get started with building, you can use the following command: + +```shell +python -m pip install httpx==0.19.0 +python manage.py distill-local build --traceback --force --collectstatic +``` + +Alternatively, you can use the [Dockerfile](./Dockerfile) in this folder. + +Both output their builds to a `build/` directory. + +> Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. + + +### Deploying To Netlify +To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: + +Build Command: +`python -m pip install httpx==0.19.0 && python static-builds/netlify_build.py` + +Publish Directory: +`build` + +Environment Variables: +- PYTHON_VERSION: 3.8 +- TOKEN: A GitHub token with access to download build artifacts. + + +Note that at this time, if you are deploying to netlify yourself, you won't have access to the +fa-icons pack we are using, which will lead to many missing icons on your preview. +You can either update the pack to one which will work on your domain, or you'll have to live with the missing icons. diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py new file mode 100644 index 00000000..6686e2ab --- /dev/null +++ b/static-builds/netlify_build.py @@ -0,0 +1,123 @@ +"""Build script to deploy project on netlify.""" + +# WARNING: This file must remain compatible with python 3.8 + +# This script performs all the actions required to build and deploy our project on netlify +# It requires the following environment variable: + +# TOKEN: A GitHub access token that can download the artifact. +# For PAT, the only scope needed is `public_repos` + +# It depends on the following packages, which are set in the netlify UI: +# httpx == 0.19.0 + +import os +import time +import zipfile +from pathlib import Path +from urllib import parse + +import httpx + +API_URL = "https://api.github.com" +OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] + + +def get_build_artifact() -> str: + """Search for a build artifact, and return the download URL.""" + print("Fetching build URL.") + + if os.getenv("PULL_REQUEST").lower() == "true": + print(f"Fetching data for PR #{os.getenv('REVIEW_ID')}") + + pull_url = f"{API_URL}/repos/{OWNER}/{REPO}/pulls/{os.getenv('REVIEW_ID')}" + pull_request = httpx.get(pull_url) + pull_request.raise_for_status() + + commit_sha = pull_request.json()["head"]["sha"] + + workflows_params = parse.urlencode({ + "event": "pull_request", + "per_page": 100 + }) + + else: + commit_sha = os.getenv("COMMIT_REF") + + workflows_params = parse.urlencode({ + "event": "push", + "per_page": 100 + }) + + print(f"Fetching action data for commit {commit_sha}") + + workflows = httpx.get(f"{API_URL}/repos/{OWNER}/{REPO}/actions/runs?{workflows_params}") + workflows.raise_for_status() + + for run in workflows.json()["workflow_runs"]: + if run["name"] == "Build & Publish Static Preview" and commit_sha == run["head_sha"]: + print(f"Found action for this commit: {run['id']}\n{run['html_url']}") + break + else: + raise Exception("Could not find the workflow run for this event.") + + polls = 0 + while polls <= 20: + if run["status"] != "completed": + print("Action isn't completed, sleeping for 30 seconds.") + polls += 1 + time.sleep(30) + + elif run["conclusion"] != "success": + print("Aborting build due to a failure in a previous CI step.") + exit(0) + + else: + print(f"Found artifact URL:\n{run['artifacts_url']}") + return run["artifacts_url"] + + _run = httpx.get(run["url"]) + _run.raise_for_status() + run = _run.json() + + raise Exception("Polled for the artifact workflow, but it was not ready in time.") + + +def download_artifact(url: str) -> None: + """Download a build artifact from `url`, and unzip the content.""" + print("Fetching artifact data.") + + artifacts = httpx.get(url) + artifacts.raise_for_status() + artifacts = artifacts.json() + + if artifacts["total_count"] == "0": + raise Exception(f"No artifacts were found for this build, aborting.\n{url}") + + for artifact in artifacts["artifacts"]: + if artifact["name"] == "static-build": + print("Found artifact with build.") + break + else: + raise Exception("Could not find an artifact with the expected name.") + + zipped_content = httpx.get(artifact["archive_download_url"], headers={ + "Authorization": f"token {os.getenv('TOKEN')}" + }) + zipped_content.raise_for_status() + + zip_file = Path("temp.zip") + zip_file.write_bytes(zipped_content.read()) + + with zipfile.ZipFile(zip_file, "r") as zip_ref: + zip_ref.extractall("build") + + zip_file.unlink(missing_ok=True) + + print("Wrote artifact content to target directory.") + + +if __name__ == "__main__": + print("Build started") + artifact_url = get_build_artifact() + download_artifact(artifact_url) -- cgit v1.2.3 From cf199b2b84773d568d7f548fdbf6ba12f63171c3 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 13:23:02 +0300 Subject: Fixes Caching On Docker Build Reworks the docker build action to use buildx in all steps to make the caching work. Reduces the wait time on the fetch action. Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 32 +++++++++++++++++++------------- static-builds/netlify_build.py | 4 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) (limited to 'static-builds') diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 50deed6d..970fad99 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -33,7 +33,7 @@ jobs: # Build the container, including an inline cache manifest to # allow us to use the registry as a cache source. - - name: Build & Push Base Image + - name: Build Docker Image (Main) uses: docker/build-push-action@v2 if: github.ref == 'refs/heads/main' with: @@ -48,22 +48,28 @@ jobs: build-args: | git_sha=${{ github.sha }} - - name: Build Local Docker Image + - name: Extract Build From Docker Image (Main) + if: github.ref == 'refs/heads/main' run: | - docker build \ - --build-arg git_sha=${{ github.sha }} \ - --cache-from ghcr.io/python-discord/static-site:latest \ - -t static-site:${{ steps.sha_tag.outputs.tag }} \ - -f static-builds/Dockerfile \ - . + mkdir docker_build \ + && docker run --name site ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker cp site:/app docker_build/ - - name: Extract Build From Docker Image - run: | - docker run --name site static-site:${{ steps.sha_tag.outputs.tag }} \ - && docker cp site:/app/build build/ + # Build directly to a local folder + - name: Build Docker Image (PR) + uses: docker/build-push-action@v2 + if: github.ref != 'refs/heads/main' + with: + context: . + file: ./static-builds/Dockerfile + push: false + cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest + outputs: type=local,dest=docker_build/ + build-args: | + git_sha=${{ github.sha }} - name: Upload Build uses: actions/upload-artifact@v2 with: name: static-build - path: build/ + path: docker_build/app/build/ diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 6686e2ab..5699c3e4 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -64,9 +64,9 @@ def get_build_artifact() -> str: polls = 0 while polls <= 20: if run["status"] != "completed": - print("Action isn't completed, sleeping for 30 seconds.") + print("Action isn't ready, sleeping for 10 seconds.") polls += 1 - time.sleep(30) + time.sleep(10) elif run["conclusion"] != "success": print("Aborting build due to a failure in a previous CI step.") -- cgit v1.2.3 From 79259eb856ea2847a61c44dee4eb03f557a7e4f1 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 14:42:40 +0300 Subject: Merges Dockerfiles Merges the normal dockerfile with the static build one to reduce duplication. Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 7 ++++--- Dockerfile | 6 ++++++ static-builds/Dockerfile | 28 ---------------------------- static-builds/README.md | 2 +- 4 files changed, 11 insertions(+), 32 deletions(-) delete mode 100644 static-builds/Dockerfile (limited to 'static-builds') diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 970fad99..06192f02 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -38,7 +38,6 @@ jobs: if: github.ref == 'refs/heads/main' with: context: . - file: ./static-builds/Dockerfile push: true cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest cache-to: type=inline @@ -47,12 +46,14 @@ jobs: ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} build-args: | git_sha=${{ github.sha }} + STATIC_BUILD=TRUE - name: Extract Build From Docker Image (Main) if: github.ref == 'refs/heads/main' run: | mkdir docker_build \ - && docker run --name site ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker run --entrypoint /bin/echo --name site \ + ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ && docker cp site:/app docker_build/ # Build directly to a local folder @@ -61,12 +62,12 @@ jobs: if: github.ref != 'refs/heads/main' with: context: . - file: ./static-builds/Dockerfile push: false cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest outputs: type=local,dest=docker_build/ build-args: | git_sha=${{ github.sha }} + STATIC_BUILD=TRUE - name: Upload Build uses: actions/upload-artifact@v2 diff --git a/Dockerfile b/Dockerfile index 046e7f80..2b039fab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,12 @@ RUN \ METRICITY_DB_URL=postgres://localhost \ python manage.py collectstatic --noinput --clear +# Build static files if we are doing a static build +ARG STATIC_BUILD=false +RUN if [ $STATIC_BUILD = "TRUE" ] ; \ + then SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force ; \ +fi + # Run web server through custom manager ENTRYPOINT ["python", "manage.py"] CMD ["run"] diff --git a/static-builds/Dockerfile b/static-builds/Dockerfile deleted file mode 100644 index a70165a0..00000000 --- a/static-builds/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Build a static preview of the site -FROM --platform=linux/amd64 python:3.9-slim-buster - -# Allow service to handle stops gracefully -STOPSIGNAL SIGQUIT - -# Set pip to have cleaner logs and no saved cache -ENV PIP_NO_CACHE_DIR=false \ - POETRY_VIRTUALENVS_CREATE=false - -# Install poetry -RUN pip install -U poetry - -# Copy the project files into working directory -WORKDIR /app - -# Install project dependencies -COPY pyproject.toml poetry.lock ./ -RUN poetry install --no-dev - -ARG git_sha="development" -ENV GIT_SHA=$git_sha -ENV PARENT_HOST=replace_me.host - -# Copy the source code in last to optimize rebuilding the image -COPY . . - -RUN SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force --collectstatic diff --git a/static-builds/README.md b/static-builds/README.md index ee2c0910..fe24df07 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -21,7 +21,7 @@ python -m pip install httpx==0.19.0 python manage.py distill-local build --traceback --force --collectstatic ``` -Alternatively, you can use the [Dockerfile](./Dockerfile) in this folder. +Alternatively, you can use the [Dockerfile](/Dockerfile) and extract the build. Both output their builds to a `build/` directory. -- cgit v1.2.3 From 036f1b6544a3f7bf8fddd3c71d42eebca784ea98 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 20:06:19 +0300 Subject: Uses Nightly To Download Artifacts Signed-off-by: Hassan Abouelela --- static-builds/README.md | 3 ++- static-builds/netlify_build.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'static-builds') diff --git a/static-builds/README.md b/static-builds/README.md index fe24df07..b5cba896 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -27,6 +27,8 @@ Both output their builds to a `build/` directory. > Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. +Note: The build script uses [nightly.link](https://github.com/oprypin/nightly.link) +to fetch the artifact with no verification. ### Deploying To Netlify To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: @@ -39,7 +41,6 @@ Publish Directory: Environment Variables: - PYTHON_VERSION: 3.8 -- TOKEN: A GitHub token with access to download build artifacts. Note that at this time, if you are deploying to netlify yourself, you won't have access to the diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 5699c3e4..4e1e6106 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -3,16 +3,12 @@ # WARNING: This file must remain compatible with python 3.8 # This script performs all the actions required to build and deploy our project on netlify -# It requires the following environment variable: - -# TOKEN: A GitHub access token that can download the artifact. -# For PAT, the only scope needed is `public_repos` - # It depends on the following packages, which are set in the netlify UI: # httpx == 0.19.0 import os import time +import typing import zipfile from pathlib import Path from urllib import parse @@ -20,11 +16,16 @@ from urllib import parse import httpx API_URL = "https://api.github.com" +NIGHTLY_URL = "https://nightly.link" OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] -def get_build_artifact() -> str: - """Search for a build artifact, and return the download URL.""" +def get_build_artifact() -> typing.Tuple[int, str]: + """ + Search for a build artifact, and return the result. + + The return is a tuple of the check suite ID, and the URL to the artifacts. + """ print("Fetching build URL.") if os.getenv("PULL_REQUEST").lower() == "true": @@ -74,7 +75,7 @@ def get_build_artifact() -> str: else: print(f"Found artifact URL:\n{run['artifacts_url']}") - return run["artifacts_url"] + return run["check_suite_id"], run["artifacts_url"] _run = httpx.get(run["url"]) _run.raise_for_status() @@ -83,7 +84,7 @@ def get_build_artifact() -> str: raise Exception("Polled for the artifact workflow, but it was not ready in time.") -def download_artifact(url: str) -> None: +def download_artifact(suite_id: int, url: str) -> None: """Download a build artifact from `url`, and unzip the content.""" print("Fetching artifact data.") @@ -101,9 +102,8 @@ def download_artifact(url: str) -> None: else: raise Exception("Could not find an artifact with the expected name.") - zipped_content = httpx.get(artifact["archive_download_url"], headers={ - "Authorization": f"token {os.getenv('TOKEN')}" - }) + artifact_url = f"{NIGHTLY_URL}/{OWNER}/{REPO}/suites/{suite_id}/artifacts/{artifact['id']}" + zipped_content = httpx.get(artifact_url) zipped_content.raise_for_status() zip_file = Path("temp.zip") @@ -119,5 +119,4 @@ def download_artifact(url: str) -> None: if __name__ == "__main__": print("Build started") - artifact_url = get_build_artifact() - download_artifact(artifact_url) + download_artifact(*get_build_artifact()) -- cgit v1.2.3