From f9ba4a8bca526cb66b513622b987f775aa88403e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 01:31:36 +0300 Subject: Adds Netlify Builds Adds an action which builds and uploads the static site as an artifact, and a fetch script to be run on the netlify builders. --- static-builds/netlify_build.py | 123 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 static-builds/netlify_build.py (limited to 'static-builds/netlify_build.py') diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py new file mode 100644 index 00000000..6686e2ab --- /dev/null +++ b/static-builds/netlify_build.py @@ -0,0 +1,123 @@ +"""Build script to deploy project on netlify.""" + +# WARNING: This file must remain compatible with python 3.8 + +# This script performs all the actions required to build and deploy our project on netlify +# It requires the following environment variable: + +# TOKEN: A GitHub access token that can download the artifact. +# For PAT, the only scope needed is `public_repos` + +# It depends on the following packages, which are set in the netlify UI: +# httpx == 0.19.0 + +import os +import time +import zipfile +from pathlib import Path +from urllib import parse + +import httpx + +API_URL = "https://api.github.com" +OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] + + +def get_build_artifact() -> str: + """Search for a build artifact, and return the download URL.""" + print("Fetching build URL.") + + if os.getenv("PULL_REQUEST").lower() == "true": + print(f"Fetching data for PR #{os.getenv('REVIEW_ID')}") + + pull_url = f"{API_URL}/repos/{OWNER}/{REPO}/pulls/{os.getenv('REVIEW_ID')}" + pull_request = httpx.get(pull_url) + pull_request.raise_for_status() + + commit_sha = pull_request.json()["head"]["sha"] + + workflows_params = parse.urlencode({ + "event": "pull_request", + "per_page": 100 + }) + + else: + commit_sha = os.getenv("COMMIT_REF") + + workflows_params = parse.urlencode({ + "event": "push", + "per_page": 100 + }) + + print(f"Fetching action data for commit {commit_sha}") + + workflows = httpx.get(f"{API_URL}/repos/{OWNER}/{REPO}/actions/runs?{workflows_params}") + workflows.raise_for_status() + + for run in workflows.json()["workflow_runs"]: + if run["name"] == "Build & Publish Static Preview" and commit_sha == run["head_sha"]: + print(f"Found action for this commit: {run['id']}\n{run['html_url']}") + break + else: + raise Exception("Could not find the workflow run for this event.") + + polls = 0 + while polls <= 20: + if run["status"] != "completed": + print("Action isn't completed, sleeping for 30 seconds.") + polls += 1 + time.sleep(30) + + elif run["conclusion"] != "success": + print("Aborting build due to a failure in a previous CI step.") + exit(0) + + else: + print(f"Found artifact URL:\n{run['artifacts_url']}") + return run["artifacts_url"] + + _run = httpx.get(run["url"]) + _run.raise_for_status() + run = _run.json() + + raise Exception("Polled for the artifact workflow, but it was not ready in time.") + + +def download_artifact(url: str) -> None: + """Download a build artifact from `url`, and unzip the content.""" + print("Fetching artifact data.") + + artifacts = httpx.get(url) + artifacts.raise_for_status() + artifacts = artifacts.json() + + if artifacts["total_count"] == "0": + raise Exception(f"No artifacts were found for this build, aborting.\n{url}") + + for artifact in artifacts["artifacts"]: + if artifact["name"] == "static-build": + print("Found artifact with build.") + break + else: + raise Exception("Could not find an artifact with the expected name.") + + zipped_content = httpx.get(artifact["archive_download_url"], headers={ + "Authorization": f"token {os.getenv('TOKEN')}" + }) + zipped_content.raise_for_status() + + zip_file = Path("temp.zip") + zip_file.write_bytes(zipped_content.read()) + + with zipfile.ZipFile(zip_file, "r") as zip_ref: + zip_ref.extractall("build") + + zip_file.unlink(missing_ok=True) + + print("Wrote artifact content to target directory.") + + +if __name__ == "__main__": + print("Build started") + artifact_url = get_build_artifact() + download_artifact(artifact_url) -- cgit v1.2.3 From cf199b2b84773d568d7f548fdbf6ba12f63171c3 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 13:23:02 +0300 Subject: Fixes Caching On Docker Build Reworks the docker build action to use buildx in all steps to make the caching work. Reduces the wait time on the fetch action. Signed-off-by: Hassan Abouelela --- .github/workflows/static-preview.yaml | 32 +++++++++++++++++++------------- static-builds/netlify_build.py | 4 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) (limited to 'static-builds/netlify_build.py') diff --git a/.github/workflows/static-preview.yaml b/.github/workflows/static-preview.yaml index 50deed6d..970fad99 100644 --- a/.github/workflows/static-preview.yaml +++ b/.github/workflows/static-preview.yaml @@ -33,7 +33,7 @@ jobs: # Build the container, including an inline cache manifest to # allow us to use the registry as a cache source. - - name: Build & Push Base Image + - name: Build Docker Image (Main) uses: docker/build-push-action@v2 if: github.ref == 'refs/heads/main' with: @@ -48,22 +48,28 @@ jobs: build-args: | git_sha=${{ github.sha }} - - name: Build Local Docker Image + - name: Extract Build From Docker Image (Main) + if: github.ref == 'refs/heads/main' run: | - docker build \ - --build-arg git_sha=${{ github.sha }} \ - --cache-from ghcr.io/python-discord/static-site:latest \ - -t static-site:${{ steps.sha_tag.outputs.tag }} \ - -f static-builds/Dockerfile \ - . + mkdir docker_build \ + && docker run --name site ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \ + && docker cp site:/app docker_build/ - - name: Extract Build From Docker Image - run: | - docker run --name site static-site:${{ steps.sha_tag.outputs.tag }} \ - && docker cp site:/app/build build/ + # Build directly to a local folder + - name: Build Docker Image (PR) + uses: docker/build-push-action@v2 + if: github.ref != 'refs/heads/main' + with: + context: . + file: ./static-builds/Dockerfile + push: false + cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest + outputs: type=local,dest=docker_build/ + build-args: | + git_sha=${{ github.sha }} - name: Upload Build uses: actions/upload-artifact@v2 with: name: static-build - path: build/ + path: docker_build/app/build/ diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 6686e2ab..5699c3e4 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -64,9 +64,9 @@ def get_build_artifact() -> str: polls = 0 while polls <= 20: if run["status"] != "completed": - print("Action isn't completed, sleeping for 30 seconds.") + print("Action isn't ready, sleeping for 10 seconds.") polls += 1 - time.sleep(30) + time.sleep(10) elif run["conclusion"] != "success": print("Aborting build due to a failure in a previous CI step.") -- cgit v1.2.3 From 036f1b6544a3f7bf8fddd3c71d42eebca784ea98 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 10 Oct 2021 20:06:19 +0300 Subject: Uses Nightly To Download Artifacts Signed-off-by: Hassan Abouelela --- static-builds/README.md | 3 ++- static-builds/netlify_build.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'static-builds/netlify_build.py') diff --git a/static-builds/README.md b/static-builds/README.md index fe24df07..b5cba896 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -27,6 +27,8 @@ Both output their builds to a `build/` directory. > Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. +Note: The build script uses [nightly.link](https://github.com/oprypin/nightly.link) +to fetch the artifact with no verification. ### Deploying To Netlify To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: @@ -39,7 +41,6 @@ Publish Directory: Environment Variables: - PYTHON_VERSION: 3.8 -- TOKEN: A GitHub token with access to download build artifacts. Note that at this time, if you are deploying to netlify yourself, you won't have access to the diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 5699c3e4..4e1e6106 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -3,16 +3,12 @@ # WARNING: This file must remain compatible with python 3.8 # This script performs all the actions required to build and deploy our project on netlify -# It requires the following environment variable: - -# TOKEN: A GitHub access token that can download the artifact. -# For PAT, the only scope needed is `public_repos` - # It depends on the following packages, which are set in the netlify UI: # httpx == 0.19.0 import os import time +import typing import zipfile from pathlib import Path from urllib import parse @@ -20,11 +16,16 @@ from urllib import parse import httpx API_URL = "https://api.github.com" +NIGHTLY_URL = "https://nightly.link" OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] -def get_build_artifact() -> str: - """Search for a build artifact, and return the download URL.""" +def get_build_artifact() -> typing.Tuple[int, str]: + """ + Search for a build artifact, and return the result. + + The return is a tuple of the check suite ID, and the URL to the artifacts. + """ print("Fetching build URL.") if os.getenv("PULL_REQUEST").lower() == "true": @@ -74,7 +75,7 @@ def get_build_artifact() -> str: else: print(f"Found artifact URL:\n{run['artifacts_url']}") - return run["artifacts_url"] + return run["check_suite_id"], run["artifacts_url"] _run = httpx.get(run["url"]) _run.raise_for_status() @@ -83,7 +84,7 @@ def get_build_artifact() -> str: raise Exception("Polled for the artifact workflow, but it was not ready in time.") -def download_artifact(url: str) -> None: +def download_artifact(suite_id: int, url: str) -> None: """Download a build artifact from `url`, and unzip the content.""" print("Fetching artifact data.") @@ -101,9 +102,8 @@ def download_artifact(url: str) -> None: else: raise Exception("Could not find an artifact with the expected name.") - zipped_content = httpx.get(artifact["archive_download_url"], headers={ - "Authorization": f"token {os.getenv('TOKEN')}" - }) + artifact_url = f"{NIGHTLY_URL}/{OWNER}/{REPO}/suites/{suite_id}/artifacts/{artifact['id']}" + zipped_content = httpx.get(artifact_url) zipped_content.raise_for_status() zip_file = Path("temp.zip") @@ -119,5 +119,4 @@ def download_artifact(url: str) -> None: if __name__ == "__main__": print("Build started") - artifact_url = get_build_artifact() - download_artifact(artifact_url) + download_artifact(*get_build_artifact()) -- cgit v1.2.3