diff options
23 files changed, 805 insertions, 209 deletions
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f97cd758..699f85ee 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -10,76 +10,26 @@ on: jobs: lint-test: runs-on: ubuntu-latest - env: - # Configure pip to cache dependencies and do a user install - PIP_NO_CACHE_DIR: false - PIP_USER: 1 - - # Make sure package manager does not use virtualenv - POETRY_VIRTUALENVS_CREATE: false - - # Specify explicit paths for python dependencies and the pre-commit - # environment so we know which directories to cache - POETRY_CACHE_DIR: ${{ github.workspace }}/.cache/py-user-base - PYTHONUSERBASE: ${{ github.workspace }}/.cache/py-user-base - PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit-cache steps: - - name: Add custom PYTHONUSERBASE to PATH - run: echo '${{ env.PYTHONUSERBASE }}/bin/' >> $GITHUB_PATH - - name: Checkout repository uses: actions/checkout@v2 - - name: Setup python - id: python - uses: actions/setup-python@v2 + - name: Install Python Dependencies + uses: HassanAbouelela/actions/setup-python@setup-python_v1.3.1 with: - python-version: '3.9' + dev: true + python_version: '3.9' # Start the database early to give it a chance to get ready before # we start running tests. - name: Run database using docker-compose run: docker-compose run -d -p 7777:5432 --name pydis_web postgres - # This step caches our Python dependencies. To make sure we - # only restore a cache when the dependencies, the python version, - # the runner operating system, and the dependency location haven't - # changed, we create a cache key that is a composite of those states. - # - # Only when the context is exactly the same, we will restore the cache. - - name: Python Dependency Caching - uses: actions/cache@v2 - id: python_cache - with: - path: ${{ env.PYTHONUSERBASE }} - key: "python-0-${{ runner.os }}-${{ env.PYTHONUSERBASE }}-\ - ${{ steps.python.outputs.python-version }}-\ - ${{ hashFiles('./pyproject.toml', './poetry.lock') }}" - - # Install our dependencies if we did not restore a dependency cache - - name: Install dependencies using poetry - if: steps.python_cache.outputs.cache-hit != 'true' - run: | - pip install poetry - poetry install - - # This step caches our pre-commit environment. To make sure we - # do create a new environment when our pre-commit setup changes, - # we create a cache key based on relevant factors. - - name: Pre-commit Environment Caching - uses: actions/cache@v2 - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: "precommit-0-${{ runner.os }}-${{ env.PRE_COMMIT_HOME }}-\ - ${{ steps.python.outputs.python-version }}-\ - ${{ hashFiles('./.pre-commit-config.yaml') }}" - # We will not run `flake8` here, as we will use a separate flake8 - # action. As pre-commit does not support user installs, we set - # PIP_USER=0 to not do a user install. + # action. - name: Run pre-commit hooks - run: export PIP_USER=0; SKIP=flake8 pre-commit run --all-files + run: SKIP=flake8 pre-commit run --all-files # Run flake8 and have it format the linting errors in the format of # the GitHub Workflow command to register error annotations. This @@ -97,7 +47,6 @@ jobs: - name: Migrations and run tests with coverage.py run: | python manage.py makemigrations --check - python manage.py migrate coverage run manage.py test --no-input coverage report -m env: @@ -132,3 +132,6 @@ log.* # Mac/OSX .DS_Store + +# Private keys +*.pem @@ -1,21 +1,14 @@ -FROM --platform=linux/amd64 python:3.9-slim-buster +FROM ghcr.io/chrislovering/python-poetry-base:3.9-slim # Allow service to handle stops gracefully STOPSIGNAL SIGQUIT -# Set pip to have cleaner logs and no saved cache -ENV PIP_NO_CACHE_DIR=false \ - POETRY_VIRTUALENVS_CREATE=false - -# Install poetry -RUN pip install -U poetry - # Copy the project files into working directory WORKDIR /app # Install project dependencies COPY pyproject.toml poetry.lock ./ -RUN poetry install --no-dev +RUN poetry install --without dev # Set Git SHA environment variable ARG git_sha="development" @@ -34,14 +27,14 @@ RUN \ SECRET_KEY=dummy_value \ DATABASE_URL=postgres://localhost \ METRICITY_DB_URL=postgres://localhost \ - python manage.py collectstatic --noinput --clear + poetry run python manage.py collectstatic --noinput --clear # Build static files if we are doing a static build ARG STATIC_BUILD=false RUN if [ $STATIC_BUILD = "TRUE" ] ; \ - then SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force ; \ + then SECRET_KEY=dummy_value poetry run python manage.py distill-local build --traceback --force ; \ fi # Run web server through custom manager -ENTRYPOINT ["python", "manage.py"] +ENTRYPOINT ["poetry", "run", "python", "manage.py"] CMD ["run"] diff --git a/docker-compose.yml b/docker-compose.yml index eb987624..a6f4fd18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,12 @@ # and additionally use the Django development server which is # unsuitable for production. -version: "3.6" +version: "3.8" services: postgres: image: postgres:13-alpine ports: - - "127.0.0.1:7777:5432" + - "7777:5432" environment: POSTGRES_DB: pysite POSTGRES_PASSWORD: pysite @@ -38,7 +38,7 @@ services: - admin.web - staff.web ports: - - "127.0.0.1:8000:8000" + - "8000:8000" depends_on: postgres: condition: service_healthy @@ -95,13 +95,15 @@ class SiteManager: name="pythondiscord.local:8000" ) - def prepare_server(self) -> None: - """Perform preparation tasks before running the server.""" + def prepare_environment(self) -> None: + """Perform common preparation tasks.""" django.setup() print("Applying migrations.") call_command("migrate", verbosity=self.verbosity) + def prepare_server(self) -> None: + """Preform runserver-specific preparation tasks.""" if self.debug: # In Production, collectstatic is ran in the Docker image print("Collecting static files.") @@ -121,6 +123,7 @@ class SiteManager: # Prevent preparing twice when in dev mode due to reloader if not self.debug or in_reloader: + self.prepare_environment() self.prepare_server() print("Starting server.") @@ -148,6 +151,11 @@ class SiteManager: # Run gunicorn for the production server. gunicorn.app.wsgiapp.run() + def run_tests(self) -> None: + """Prepare and run the test suite.""" + self.prepare_environment() + call_command(*sys.argv[1:]) + def clean_up_static_files(build_folder: Path) -> None: """Recursively loop over the build directory and fix links.""" @@ -168,8 +176,12 @@ def clean_up_static_files(build_folder: Path) -> None: def main() -> None: """Entry point for Django management script.""" # Use the custom site manager for launching the server - if len(sys.argv) > 1 and sys.argv[1] == "run": - SiteManager(sys.argv).run_server() + if len(sys.argv) > 1 and sys.argv[1] in ("run", "test"): + manager = SiteManager(sys.argv) + if sys.argv[1] == "run": + manager.run_server() + elif sys.argv[1] == "test": + manager.run_tests() # Pass any others directly to standard management commands else: diff --git a/poetry.lock b/poetry.lock index f6576fba..1bee4397 100644 --- a/poetry.lock +++ b/poetry.lock @@ -68,6 +68,17 @@ optional = false python-versions = ">=3.6" [[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." @@ -122,6 +133,25 @@ requests = ">=1.0.0" yaml = ["PyYAML (>=3.10)"] [[package]] +name = "cryptography" +version = "37.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools_rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + +[[package]] name = "distlib" version = "0.3.4" description = "Distribution utilities" @@ -608,6 +638,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] name = "pydocstyle" version = "6.1.1" description = "Python docstring style checker" @@ -638,6 +676,23 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] +name = "pyjwt" +version = "2.4.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cryptography = {version = ">=3.3.1", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.3.1)"] +dev = ["sphinx", "sphinx-rtd-theme", "zope.interface", "cryptography (>=3.3.1)", "pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)", "mypy", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"] + +[[package]] name = "python-dotenv" version = "0.17.1" description = "Read key-value pairs from a .env file and set them as environment variables" @@ -876,7 +931,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "e71d10c3d478c5d99e842f4c449a093caa1d4b2d255eb0dfb19843c5265d4aca" +content-hash = "c656c07f40d32ee7d30c19a7084b40e1e851209a362a3fe882aa03c2fd286454" [metadata.files] anyio = [ @@ -896,6 +951,7 @@ certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] +cffi = [] cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, @@ -963,6 +1019,7 @@ coveralls = [ {file = "coveralls-2.2.0-py2.py3-none-any.whl", hash = "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc"}, {file = "coveralls-2.2.0.tar.gz", hash = "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617"}, ] +cryptography = [] distlib = [ {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, @@ -1157,6 +1214,10 @@ pycodestyle = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, ] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, @@ -1166,6 +1227,10 @@ pyflakes = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] +pyjwt = [ + {file = "PyJWT-2.4.0-py3-none-any.whl", hash = "sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf"}, + {file = "PyJWT-2.4.0.tar.gz", hash = "sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba"}, +] python-dotenv = [ {file = "python-dotenv-0.17.1.tar.gz", hash = "sha256:b1ae5e9643d5ed987fc57cc2583021e38db531946518130777734f9589b3141f"}, {file = "python_dotenv-0.17.1-py2.py3-none-any.whl", hash = "sha256:00aa34e92d992e9f8383730816359647f358f4a3be1ba45e5a5cefd27ee91544"}, diff --git a/pydis_site/apps/api/github_utils.py b/pydis_site/apps/api/github_utils.py new file mode 100644 index 00000000..5d7bcdc3 --- /dev/null +++ b/pydis_site/apps/api/github_utils.py @@ -0,0 +1,209 @@ +"""Utilities for working with the GitHub API.""" +import dataclasses +import datetime +import math +import typing + +import httpx +import jwt + +from pydis_site import settings + +MAX_RUN_TIME = datetime.timedelta(minutes=10) +"""The maximum time allowed before an action is declared timed out.""" +ISO_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" +"""The datetime string format GitHub uses.""" + + +class ArtifactProcessingError(Exception): + """Base exception for other errors related to processing a GitHub artifact.""" + + status: int + + +class UnauthorizedError(ArtifactProcessingError): + """The application does not have permission to access the requested repo.""" + + status = 401 + + +class NotFoundError(ArtifactProcessingError): + """The requested resource could not be found.""" + + status = 404 + + +class ActionFailedError(ArtifactProcessingError): + """The requested workflow did not conclude successfully.""" + + status = 400 + + +class RunTimeoutError(ArtifactProcessingError): + """The requested workflow run was not ready in time.""" + + status = 408 + + +class RunPendingError(ArtifactProcessingError): + """The requested workflow run is still pending, try again later.""" + + status = 202 + + [email protected](frozen=True) +class WorkflowRun: + """ + A workflow run from the GitHub API. + + https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run + """ + + name: str + head_sha: str + created_at: str + status: str + conclusion: str + artifacts_url: str + + @classmethod + def from_raw(cls, data: dict[str, typing.Any]): + """Create an instance using the raw data from the API, discarding unused fields.""" + return cls(**{ + key.name: data[key.name] for key in dataclasses.fields(cls) + }) + + +def generate_token() -> str: + """ + Generate a JWT token to access the GitHub API. + + The token is valid for roughly 10 minutes after generation, before the API starts + returning 401s. + + Refer to: + https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app + """ + now = datetime.datetime.now() + return jwt.encode( + { + "iat": math.floor((now - datetime.timedelta(seconds=60)).timestamp()), # Issued at + "exp": math.floor((now + datetime.timedelta(minutes=9)).timestamp()), # Expires at + "iss": settings.GITHUB_APP_ID, + }, + settings.GITHUB_APP_KEY, + algorithm="RS256" + ) + + +def authorize(owner: str, repo: str) -> httpx.Client: + """ + Get an access token for the requested repository. + + The process is roughly: + - GET app/installations to get a list of all app installations + - POST <app_access_token> to get a token to access the given app + - GET installation/repositories and check if the requested one is part of those + """ + client = httpx.Client( + base_url=settings.GITHUB_API, + headers={"Authorization": f"bearer {generate_token()}"}, + timeout=settings.TIMEOUT_PERIOD, + ) + + try: + # Get a list of app installations we have access to + apps = client.get("app/installations") + apps.raise_for_status() + + for app in apps.json(): + # Look for an installation with the right owner + if app["account"]["login"] != owner: + continue + + # Get the repositories of the specified owner + app_token = client.post(app["access_tokens_url"]) + app_token.raise_for_status() + client.headers["Authorization"] = f"bearer {app_token.json()['token']}" + + repos = client.get("installation/repositories") + repos.raise_for_status() + + # Search for the request repository + for accessible_repo in repos.json()["repositories"]: + if accessible_repo["name"] == repo: + # We've found the correct repository, and it's accessible with the current auth + return client + + raise NotFoundError( + "Could not find the requested repository. Make sure the application can access it." + ) + + except BaseException as e: + # Close the client if we encountered an unexpected exception + client.close() + raise e + + +def check_run_status(run: WorkflowRun) -> str: + """Check if the provided run has been completed, otherwise raise an exception.""" + created_at = datetime.datetime.strptime(run.created_at, ISO_FORMAT_STRING) + run_time = datetime.datetime.utcnow() - created_at + + if run.status != "completed": + if run_time <= MAX_RUN_TIME: + raise RunPendingError( + f"The requested run is still pending. It was created " + f"{run_time.seconds // 60}:{run_time.seconds % 60 :>02} minutes ago." + ) + else: + raise RunTimeoutError("The requested workflow was not ready in time.") + + if run.conclusion != "success": + # The action failed, or did not run + raise ActionFailedError(f"The requested workflow ended with: {run.conclusion}") + + # The requested action is ready + return run.artifacts_url + + +def get_artifact(owner: str, repo: str, sha: str, action_name: str, artifact_name: str) -> str: + """Get a download URL for a build artifact.""" + client = authorize(owner, repo) + + try: + # Get the workflow runs for this repository + runs = client.get(f"/repos/{owner}/{repo}/actions/runs", params={"per_page": 100}) + runs.raise_for_status() + runs = runs.json() + + # Filter the runs for the one associated with the given SHA + for run in runs["workflow_runs"]: + run = WorkflowRun.from_raw(run) + if run.name == action_name and sha == run.head_sha: + break + else: + raise NotFoundError( + "Could not find a run matching the provided settings in the previous hundred runs." + ) + + # Check the workflow status + url = check_run_status(run) + + # Filter the artifacts, and return the download URL + artifacts = client.get(url) + artifacts.raise_for_status() + + for artifact in artifacts.json()["artifacts"]: + if artifact["name"] == artifact_name: + data = client.get(artifact["archive_download_url"]) + if data.status_code == 302: + return str(data.next_request.url) + + # The following line is left untested since it should in theory be impossible + data.raise_for_status() # pragma: no cover + + raise NotFoundError("Could not find an artifact matching the provided name.") + + finally: + client.close() diff --git a/pydis_site/apps/api/migrations/0084_infraction_last_applied.py b/pydis_site/apps/api/migrations/0084_infraction_last_applied.py new file mode 100644 index 00000000..7704ddb8 --- /dev/null +++ b/pydis_site/apps/api/migrations/0084_infraction_last_applied.py @@ -0,0 +1,26 @@ +# Generated by Django 4.0.6 on 2022-07-27 20:32 + +import django.utils.timezone +from django.db import migrations, models +from django.apps.registry import Apps + + +def set_last_applied_to_inserted_at(apps: Apps, schema_editor): + Infractions = apps.get_model("api", "infraction") + Infractions.objects.all().update(last_applied=models.F("inserted_at")) + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0083_remove_embed_validation'), + ] + + operations = [ + migrations.AddField( + model_name='infraction', + name='last_applied', + field=models.DateTimeField(default=django.utils.timezone.now, help_text='The date and time of when this infraction was last applied.'), + ), + migrations.RunPython(set_last_applied_to_inserted_at) + ] diff --git a/pydis_site/apps/api/models/bot/infraction.py b/pydis_site/apps/api/models/bot/infraction.py index c9303024..218ee5ec 100644 --- a/pydis_site/apps/api/models/bot/infraction.py +++ b/pydis_site/apps/api/models/bot/infraction.py @@ -23,6 +23,12 @@ class Infraction(ModelReprMixin, models.Model): default=timezone.now, help_text="The date and time of the creation of this infraction." ) + last_applied = models.DateTimeField( + # This default is for backwards compatibility with bot versions + # that don't explicitly give a value. + default=timezone.now, + help_text="The date and time of when this infraction was last applied." + ) expires_at = models.DateTimeField( null=True, help_text=( diff --git a/pydis_site/apps/api/serializers.py b/pydis_site/apps/api/serializers.py index e53ccffa..9228c1f4 100644 --- a/pydis_site/apps/api/serializers.py +++ b/pydis_site/apps/api/serializers.py @@ -176,6 +176,7 @@ class InfractionSerializer(ModelSerializer): fields = ( 'id', 'inserted_at', + 'last_applied', 'expires_at', 'active', 'user', diff --git a/pydis_site/apps/api/tests/test_github_utils.py b/pydis_site/apps/api/tests/test_github_utils.py new file mode 100644 index 00000000..f642f689 --- /dev/null +++ b/pydis_site/apps/api/tests/test_github_utils.py @@ -0,0 +1,285 @@ +import dataclasses +import datetime +import typing +import unittest +from unittest import mock + +import django.test +import httpx +import jwt +import rest_framework.response +import rest_framework.test +from django.urls import reverse + +from .. import github_utils + + +class GeneralUtilityTests(unittest.TestCase): + """Test the utility methods which do not fit in another class.""" + + def test_token_generation(self): + """Test that the a valid JWT token is generated.""" + def encode(payload: dict, _: str, algorithm: str, *args, **kwargs) -> str: + """ + Intercept the encode method. + + The result is encoded with an algorithm which does not require a PEM key, as it may + not be available in testing environments. + """ + self.assertEqual("RS256", algorithm, "The GitHub App JWT must be signed using RS256.") + return original_encode( + payload, "secret-encoding-key", algorithm="HS256", *args, **kwargs + ) + + original_encode = jwt.encode + with mock.patch("jwt.encode", new=encode): + token = github_utils.generate_token() + decoded = jwt.decode(token, "secret-encoding-key", algorithms=["HS256"]) + + delta = datetime.timedelta(minutes=10) + self.assertAlmostEqual(decoded["exp"] - decoded["iat"], delta.total_seconds()) + self.assertLess(decoded["exp"], (datetime.datetime.now() + delta).timestamp()) + + +class CheckRunTests(unittest.TestCase): + """Tests the check_run_status utility.""" + + run_kwargs: typing.Mapping = { + "name": "run_name", + "head_sha": "sha", + "status": "completed", + "conclusion": "success", + "created_at": datetime.datetime.utcnow().strftime(github_utils.ISO_FORMAT_STRING), + "artifacts_url": "url", + } + + def test_completed_run(self): + """Test that an already completed run returns the correct URL.""" + final_url = "some_url_string_1234" + + kwargs = dict(self.run_kwargs, artifacts_url=final_url) + result = github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + self.assertEqual(final_url, result) + + def test_pending_run(self): + """Test that a pending run raises the proper exception.""" + kwargs = dict(self.run_kwargs, status="pending") + with self.assertRaises(github_utils.RunPendingError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + def test_timeout_error(self): + """Test that a timeout is declared after a certain duration.""" + kwargs = dict(self.run_kwargs, status="pending") + # Set the creation time to well before the MAX_RUN_TIME + # to guarantee the right conclusion + kwargs["created_at"] = ( + datetime.datetime.utcnow() - github_utils.MAX_RUN_TIME - datetime.timedelta(minutes=10) + ).strftime(github_utils.ISO_FORMAT_STRING) + + with self.assertRaises(github_utils.RunTimeoutError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + def test_failed_run(self): + """Test that a failed run raises the proper exception.""" + kwargs = dict(self.run_kwargs, conclusion="failed") + with self.assertRaises(github_utils.ActionFailedError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + +def get_response_authorize(_: httpx.Client, request: httpx.Request, **__) -> httpx.Response: + """ + Helper method for the authorize tests. + + Requests are intercepted before being sent out, and the appropriate responses are returned. + """ + path = request.url.path + auth = request.headers.get("Authorization") + + if request.method == "GET": + if path == "/app/installations": + if auth == "bearer JWT initial token": + return httpx.Response(200, request=request, json=[{ + "account": {"login": "VALID_OWNER"}, + "access_tokens_url": "https://example.com/ACCESS_TOKEN_URL" + }]) + else: + return httpx.Response( + 401, json={"error": "auth app/installations"}, request=request + ) + + elif path == "/installation/repositories": + if auth == "bearer app access token": + return httpx.Response(200, request=request, json={ + "repositories": [{ + "name": "VALID_REPO" + }] + }) + else: # pragma: no cover + return httpx.Response( + 401, json={"error": "auth installation/repositories"}, request=request + ) + + elif request.method == "POST": + if path == "/ACCESS_TOKEN_URL": + if auth == "bearer JWT initial token": + return httpx.Response(200, request=request, json={"token": "app access token"}) + else: # pragma: no cover + return httpx.Response(401, json={"error": "auth access_token"}, request=request) + + # Reaching this point means something has gone wrong + return httpx.Response(500, request=request) # pragma: no cover + + [email protected]("httpx.Client.send", new=get_response_authorize) [email protected](github_utils, "generate_token", new=mock.Mock(return_value="JWT initial token")) +class AuthorizeTests(unittest.TestCase): + """Test the authorize utility.""" + + def test_invalid_apps_auth(self): + """Test that an exception is raised if authorization was attempted with an invalid token.""" + with mock.patch.object(github_utils, "generate_token", return_value="Invalid token"): + with self.assertRaises(httpx.HTTPStatusError) as error: + github_utils.authorize("VALID_OWNER", "VALID_REPO") + + exception: httpx.HTTPStatusError = error.exception + self.assertEqual(401, exception.response.status_code) + self.assertEqual("auth app/installations", exception.response.json()["error"]) + + def test_missing_repo(self): + """Test that an exception is raised when the selected owner or repo are not available.""" + with self.assertRaises(github_utils.NotFoundError): + github_utils.authorize("INVALID_OWNER", "VALID_REPO") + with self.assertRaises(github_utils.NotFoundError): + github_utils.authorize("VALID_OWNER", "INVALID_REPO") + + def test_valid_authorization(self): + """Test that an accessible repository can be accessed.""" + client = github_utils.authorize("VALID_OWNER", "VALID_REPO") + self.assertEqual("bearer app access token", client.headers.get("Authorization")) + + +class ArtifactFetcherTests(unittest.TestCase): + """Test the get_artifact utility.""" + + @staticmethod + def get_response_get_artifact(request: httpx.Request, **_) -> httpx.Response: + """ + Helper method for the get_artifact tests. + + Requests are intercepted before being sent out, and the appropriate responses are returned. + """ + path = request.url.path + + if "force_error" in path: + return httpx.Response(404, request=request) + + if request.method == "GET": + if path == "/repos/owner/repo/actions/runs": + run = github_utils.WorkflowRun( + name="action_name", + head_sha="action_sha", + created_at=datetime.datetime.now().strftime(github_utils.ISO_FORMAT_STRING), + status="completed", + conclusion="success", + artifacts_url="artifacts_url" + ) + return httpx.Response( + 200, request=request, json={"workflow_runs": [dataclasses.asdict(run)]} + ) + elif path == "/artifact_url": + return httpx.Response( + 200, request=request, json={"artifacts": [{ + "name": "artifact_name", + "archive_download_url": "artifact_download_url" + }]} + ) + elif path == "/artifact_download_url": + response = httpx.Response(302, request=request) + response.next_request = httpx.Request( + "GET", + httpx.URL("https://final_download.url") + ) + return response + + # Reaching this point means something has gone wrong + return httpx.Response(500, request=request) # pragma: no cover + + def setUp(self) -> None: + self.call_args = ["owner", "repo", "action_sha", "action_name", "artifact_name"] + self.client = httpx.Client(base_url="https://example.com") + + self.patchers = [ + mock.patch.object(self.client, "send", new=self.get_response_get_artifact), + mock.patch.object(github_utils, "authorize", return_value=self.client), + mock.patch.object(github_utils, "check_run_status", return_value="artifact_url"), + ] + + for patcher in self.patchers: + patcher.start() + + def tearDown(self) -> None: + for patcher in self.patchers: + patcher.stop() + + def test_client_closed_on_errors(self): + """Test that the client is terminated even if an error occurs at some point.""" + self.call_args[0] = "force_error" + with self.assertRaises(httpx.HTTPStatusError): + github_utils.get_artifact(*self.call_args) + self.assertTrue(self.client.is_closed) + + def test_missing(self): + """Test that an exception is raised if the requested artifact was not found.""" + cases = ( + "invalid sha", + "invalid action name", + "invalid artifact name", + ) + for i, name in enumerate(cases, 2): + with self.subTest(f"Test {name} raises an error"): + new_args = self.call_args.copy() + new_args[i] = name + + with self.assertRaises(github_utils.NotFoundError): + github_utils.get_artifact(*new_args) + + def test_valid(self): + """Test that the correct download URL is returned for valid requests.""" + url = github_utils.get_artifact(*self.call_args) + self.assertEqual("https://final_download.url", url) + self.assertTrue(self.client.is_closed) + + [email protected](github_utils, "get_artifact") +class GitHubArtifactViewTests(django.test.TestCase): + """Test the GitHub artifact fetch API view.""" + + def setUp(self): + self.kwargs = { + "owner": "test_owner", + "repo": "test_repo", + "sha": "test_sha", + "action_name": "test_action", + "artifact_name": "test_artifact", + } + self.url = reverse("api:github-artifacts", kwargs=self.kwargs) + + def test_correct_artifact(self, artifact_mock: mock.Mock): + """Test a proper response is returned with proper input.""" + artifact_mock.return_value = "final download url" + result = self.client.get(self.url) + + self.assertIsInstance(result, rest_framework.response.Response) + self.assertEqual({"url": artifact_mock.return_value}, result.data) + + def test_failed_fetch(self, artifact_mock: mock.Mock): + """Test that a proper error is returned when the request fails.""" + artifact_mock.side_effect = github_utils.NotFoundError("Test error message") + result = self.client.get(self.url) + + self.assertIsInstance(result, rest_framework.response.Response) + self.assertEqual({ + "error_type": github_utils.NotFoundError.__name__, + "error": "Test error message", + "requested_resource": "/".join(self.kwargs.values()) + }, result.data) diff --git a/pydis_site/apps/api/tests/test_infractions.py b/pydis_site/apps/api/tests/test_infractions.py index f1107734..89ee4e23 100644 --- a/pydis_site/apps/api/tests/test_infractions.py +++ b/pydis_site/apps/api/tests/test_infractions.py @@ -56,15 +56,17 @@ class InfractionTests(AuthenticatedAPITestCase): type='ban', reason='He terk my jerb!', hidden=True, + inserted_at=dt(2020, 10, 10, 0, 0, 0, tzinfo=timezone.utc), expires_at=dt(5018, 11, 20, 15, 52, tzinfo=timezone.utc), - active=True + active=True, ) cls.ban_inactive = Infraction.objects.create( user_id=cls.user.id, actor_id=cls.user.id, type='ban', reason='James is an ass, and we won\'t be working with him again.', - active=False + active=False, + inserted_at=dt(2020, 10, 10, 0, 1, 0, tzinfo=timezone.utc), ) cls.mute_permanent = Infraction.objects.create( user_id=cls.user.id, @@ -72,7 +74,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='mute', reason='He has a filthy mouth and I am his soap.', active=True, - expires_at=None + inserted_at=dt(2020, 10, 10, 0, 2, 0, tzinfo=timezone.utc), + expires_at=None, ) cls.superstar_expires_soon = Infraction.objects.create( user_id=cls.user.id, @@ -80,7 +83,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='superstar', reason='This one doesn\'t matter anymore.', active=True, - expires_at=dt.now(timezone.utc) + datetime.timedelta(hours=5) + inserted_at=dt(2020, 10, 10, 0, 3, 0, tzinfo=timezone.utc), + expires_at=dt.now(timezone.utc) + datetime.timedelta(hours=5), ) cls.voiceban_expires_later = Infraction.objects.create( user_id=cls.user.id, @@ -88,7 +92,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='voice_ban', reason='Jet engine mic', active=True, - expires_at=dt.now(timezone.utc) + datetime.timedelta(days=5) + inserted_at=dt(2020, 10, 10, 0, 4, 0, tzinfo=timezone.utc), + expires_at=dt.now(timezone.utc) + datetime.timedelta(days=5), ) def test_list_all(self): diff --git a/pydis_site/apps/api/urls.py b/pydis_site/apps/api/urls.py index 1e564b29..2757f176 100644 --- a/pydis_site/apps/api/urls.py +++ b/pydis_site/apps/api/urls.py @@ -1,7 +1,7 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter -from .views import HealthcheckView, RulesView +from .views import GitHubArtifactsView, HealthcheckView, RulesView from .viewsets import ( AocAccountLinkViewSet, AocCompletionistBlockViewSet, @@ -86,5 +86,10 @@ urlpatterns = ( # from django_hosts.resolvers import reverse path('bot/', include((bot_router.urls, 'api'), namespace='bot')), path('healthcheck', HealthcheckView.as_view(), name='healthcheck'), - path('rules', RulesView.as_view(), name='rules') + path('rules', RulesView.as_view(), name='rules'), + path( + 'github/artifact/<str:owner>/<str:repo>/<str:sha>/<str:action_name>/<str:artifact_name>', + GitHubArtifactsView.as_view(), + name="github-artifacts" + ), ) diff --git a/pydis_site/apps/api/views.py b/pydis_site/apps/api/views.py index 816463f6..34167a38 100644 --- a/pydis_site/apps/api/views.py +++ b/pydis_site/apps/api/views.py @@ -1,7 +1,10 @@ from rest_framework.exceptions import ParseError +from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from . import github_utils + class HealthcheckView(APIView): """ @@ -34,12 +37,14 @@ class RulesView(APIView): ## Routes ### GET /rules - Returns a JSON array containing the server's rules: + Returns a JSON array containing the server's rules + and keywords relating to each rule. + Example response: >>> [ - ... "Eat candy.", - ... "Wake up at 4 AM.", - ... "Take your medicine." + ... ["Eat candy.", ["candy", "sweets"]], + ... ["Wake up at 4 AM.", ["wake_up", "early", "early_bird"]], + ... ["Take your medicine.", ["medicine", "health"]] ... ] Since some of the the rules require links, this view @@ -97,6 +102,12 @@ class RulesView(APIView): # `format` here is the result format, we have a link format here instead. def get(self, request, format=None): # noqa: D102,ANN001,ANN201 + """ + Returns a list of our community rules coupled with their keywords. + + Each item in the returned list is a tuple with the rule as first item + and a list of keywords that match that rules as second item. + """ link_format = request.query_params.get('link_format', 'md') if link_format not in ('html', 'md'): raise ParseError( @@ -121,34 +132,93 @@ class RulesView(APIView): return Response([ ( - f"Follow the {pydis_coc}." + f"Follow the {pydis_coc}.", + ["coc", "conduct", "code"] ), ( - f"Follow the {discord_community_guidelines} and {discord_tos}." + f"Follow the {discord_community_guidelines} and {discord_tos}.", + ["discord", "guidelines", "discord_tos"] ), ( - "Respect staff members and listen to their instructions." + "Respect staff members and listen to their instructions.", + ["respect", "staff", "instructions"] ), ( "Use English to the best of your ability. " - "Be polite if someone speaks English imperfectly." + "Be polite if someone speaks English imperfectly.", + ["english", "language"] ), ( "Do not provide or request help on projects that may break laws, " - "breach terms of services, or are malicious or inappropriate." + "breach terms of services, or are malicious or inappropriate.", + ["infraction", "tos", "breach", "malicious", "inappropriate"] ), ( - "Do not post unapproved advertising." + "Do not post unapproved advertising.", + ["ad", "ads", "advert", "advertising"] ), ( "Keep discussions relevant to the channel topic. " - "Each channel's description tells you the topic." + "Each channel's description tells you the topic.", + ["off-topic", "topic", "relevance"] ), ( "Do not help with ongoing exams. When helping with homework, " - "help people learn how to do the assignment without doing it for them." + "help people learn how to do the assignment without doing it for them.", + ["exam", "exams", "assignment", "assignments", "homework"] ), ( - "Do not offer or ask for paid work of any kind." + "Do not offer or ask for paid work of any kind.", + ["paid", "work", "money"] ), ]) + + +class GitHubArtifactsView(APIView): + """ + Provides utilities for interacting with the GitHub API and obtaining action artifacts. + + ## Routes + ### GET /github/artifacts + Returns a download URL for the artifact requested. + + { + 'url': 'https://pipelines.actions.githubusercontent.com/...' + } + + ### Exceptions + In case of an error, the following body will be returned: + + { + "error_type": "<error class name>", + "error": "<error description>", + "requested_resource": "<owner>/<repo>/<sha>/<artifact_name>" + } + + ## Authentication + Does not require any authentication nor permissions. + """ + + authentication_classes = () + permission_classes = () + + def get( + self, + request: Request, + *, + owner: str, + repo: str, + sha: str, + action_name: str, + artifact_name: str + ) -> Response: + """Return a download URL for the requested artifact.""" + try: + url = github_utils.get_artifact(owner, repo, sha, action_name, artifact_name) + return Response({"url": url}) + except github_utils.ArtifactProcessingError as e: + return Response({ + "error_type": e.__class__.__name__, + "error": str(e), + "requested_resource": f"{owner}/{repo}/{sha}/{action_name}/{artifact_name}" + }, status=e.status) diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md index 6231fe87..2822d046 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md @@ -119,7 +119,7 @@ As mentioned in the Contributing Guidelines, we have a simple style guide for ou [**Style Guide**](./style-guide/) ### 4. Create an issue -The first step to any new contribution is an issue describing a problem with the current codebase or proposing a new feature. All the open issues are viewable on the GitHub repositories, for instance here is the [issues page for Sir Lancebot](https://github.com/python-discord/sir-lancebot/issues). If you have something that you want to implement open a new issue to present your idea. Otherwise you can browse the unassigned issues and ask to be assigned to one that you're interested in, either in the comments on the issue or in the [`#dev-contrib`](https://discord.gg/2h3qBv8Xaa) channel on Discord. +The first step to any new contribution is an issue describing a problem with the current codebase or proposing a new feature. All the open issues are viewable on the GitHub repositories, for instance here is the [issues page for Sir Lancebot](https://github.com/python-discord/sir-lancebot/issues). If you have something that you want to implement open a new issue to present your idea. Otherwise, you can browse the unassigned issues and ask to be assigned to one that you're interested in, either in the comments on the issue or in the [`#dev-contrib`](https://discord.gg/2h3qBv8Xaa) channel on Discord. [**How to write a good issue**](./issues/) diff --git a/pydis_site/settings.py b/pydis_site/settings.py index 03c16f4b..bbf1d3aa 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -21,7 +21,6 @@ import environ import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration - env = environ.Env( DEBUG=(bool, False), SITE_DSN=(str, ""), @@ -30,10 +29,19 @@ env = environ.Env( GIT_SHA=(str, 'development'), TIMEOUT_PERIOD=(int, 5), GITHUB_TOKEN=(str, None), + GITHUB_APP_ID=(str, None), + GITHUB_APP_KEY=(str, None), ) GIT_SHA = env("GIT_SHA") +GITHUB_API = "https://api.github.com" GITHUB_TOKEN = env("GITHUB_TOKEN") +GITHUB_APP_ID = env("GITHUB_APP_ID") +GITHUB_APP_KEY = env("GITHUB_APP_KEY") + +if GITHUB_APP_KEY and (key_file := Path(GITHUB_APP_KEY)).is_file(): + # Allow the OAuth key to be loaded from a file + GITHUB_APP_KEY = key_file.read_text(encoding="utf-8") sentry_sdk.init( dsn=env('SITE_DSN'), diff --git a/pydis_site/templates/events/index.html b/pydis_site/templates/events/index.html index 796a2e34..640682d0 100644 --- a/pydis_site/templates/events/index.html +++ b/pydis_site/templates/events/index.html @@ -9,6 +9,9 @@ {% block event_content %} <div class="box"> <h2 class="title is-4"><a href="{% url "events:page" path="code-jams" %}">Code Jams</a></h2> + <div class="notification is-success"> + <a href="{% url "events:page" path="code-jams/9" %}">The <b>2022 Summer Code Jam</b> is underway!</a>. + </div> <p>Every year we hold a community-wide Summer Code Jam. For this event, members of our community are assigned to teams to collaborate and create something amazing using a technology we picked for them. One such technology that was picked for the Summer 2021 Code Jam was text user interfaces (TUIs), where teams could pick from a pre-approved list of frameworks.</p> <p>To help fuel the creative process, we provide a specific theme, like <strong>Think Inside the Box</strong> or <strong>Early Internet</strong>. At the end of the Code Jam, the projects are judged by Python Discord server staff members and guest judges from the larger Python community. The judges will consider creativity, code quality, teamwork, and adherence to the theme.</p> <p>If you want to read more about Code Jams, visit our <a href="{% url "events:page" path="code-jams" %}">Code Jam info page</a> or watch this video showcasing the best projects created during the <strong>Winter Code Jam 2020: Ancient Technology</strong>:</p> diff --git a/pydis_site/templates/events/pages/code-jams/9/_index.html b/pydis_site/templates/events/pages/code-jams/9/_index.html index 8624bfbb..ca7c4f90 100644 --- a/pydis_site/templates/events/pages/code-jams/9/_index.html +++ b/pydis_site/templates/events/pages/code-jams/9/_index.html @@ -26,9 +26,9 @@ <li><strike>Wednesday, June 29 - The Qualifier is released</strike></li> <li><strike>Wednesday, July 6 - Voting for the theme opens</strike></li> <li><strike>Wednesday, July 13 - The Qualifier closes</strike></li> - <li>Thursday, July 21 - Code Jam Begins</li> - <li>Sunday, July 31 - Coding portion of the jam ends</li> - <li>Sunday, August 4 - Code Jam submissions are closed</li> + <li><strike>Thursday, July 21 - Code Jam Begins</strike></li> + <li><strike>Sunday, July 31 - Coding portion of the jam ends</strike></li> + <li><strike>Sunday, August 4 - Code Jam submissions are closed</strike></li> </ul> <h3 id="qualifier"><a href="#how-to-join">The Qualifier</a></h3> diff --git a/pydis_site/templates/events/pages/code-jams/9/rules.html b/pydis_site/templates/events/pages/code-jams/9/rules.html index 5ad75d67..9a28852f 100644 --- a/pydis_site/templates/events/pages/code-jams/9/rules.html +++ b/pydis_site/templates/events/pages/code-jams/9/rules.html @@ -11,17 +11,17 @@ {% block event_content %} <ol> - <li><p>Your solution must use one of the approved frameworks (a list will be released soon). It is not permitted to circumvent this rule by e.g. using the approved framework as a wrapper for another framework.</p></li> + <li><p>Your solution must use one of the approved frameworks. It is not permitted to circumvent this rule by e.g. using the approved framework as a wrapper for another framework.</p></li> <li> - <p><strong>Your solution must only communicate through WebSockets</strong>. Any communication between the client and server (or any two server workers), must utilize WebSockets.</p> - <p> - An exception to this rule is that communication with databases and files is allowed for accessing resources or for storage purposes. - For example, you may use PostgreSQL as a database but not its `NOTIFY` command. - Working with subprocesses (through stdin/stdout or <code>multiprocessing.Pool()</code>/<code>concurrent.futures.ProcessPoolExecutor()</code>) is also allowed. + <strong>The core of your project must use WebSockets as its communication protocol.</strong>. + This means that you are allowed to use other methods of communication where WebSockets cannot be implemented, however, that should be a non-significant portion of your project. + For example, serving static files for a website cannot be done over WebSockets and it does not pose as a significant portion of a project, therefore it is allowed. </p> - <p>If you're interested in utilizing a particular non-WebSocket method of communication, reach out to the Events Team for discussion and approval</p> + <p>This rule does not apply to databases and files when used for <i>storage purposes</i> even though that may be a significant portion of your project. Working with subprocesses (through stdin/stdout or <code>multiprocessing.Pool()</code>/<code>concurrent.futures.ProcessPoolExecutor()</code>) is also exempt from this rule.</p> + + <p>If you're unsure about your use of non-WebSocket communication, please reach out to the events team.</p> </li> <li><p>Your solution should be platform agnostic. For example, if you use filepaths in your submission, use <code>pathlib</code> to create platform agnostic Path objects instead of hardcoding the paths.</p></li> <li> diff --git a/pydis_site/templates/events/sidebar/code-jams/previous-code-jams.html b/pydis_site/templates/events/sidebar/code-jams/previous-code-jams.html index 21b2ccb4..28412c53 100644 --- a/pydis_site/templates/events/sidebar/code-jams/previous-code-jams.html +++ b/pydis_site/templates/events/sidebar/code-jams/previous-code-jams.html @@ -1,6 +1,7 @@ <div class="box"> <p class="menu-label">Previous Code Jams</p> <ul class="menu-list"> + <li><a class="has-text-link" href="{% url "events:page" path="code-jams/9" %}">Code Jam 9: It's Not A Bug, It's A Feature</a></li> <li><a class="has-text-link" href="{% url "events:page" path="code-jams/8" %}">Code Jam 8: Think Inside the Box</a></li> <li><a class="has-text-link" href="{% url "events:page" path="code-jams/7" %}">Code Jam 7: Early Internet</a></li> <li><a class="has-text-link" href="{% url "events:page" path="code-jams/6" %}">Code Jam 6: Ancient Technology</a></li> diff --git a/pyproject.toml b/pyproject.toml index 467fc8bc..037f837c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ markdown = "~=3.3.4" python-frontmatter = "~=1.0" django-prometheus = "~=2.1" django-distill = "~=2.9.0" +PyJWT = {version = "~=2.4.0", extras = ["crypto"]} [tool.poetry.dev-dependencies] coverage = "~=5.0" @@ -51,7 +52,7 @@ start = "python manage.py run --debug" makemigrations = "python manage.py makemigrations" django_shell = "python manage.py shell" test = "coverage run manage.py test" -coverage = "coverage run manage.py test --no-input; coverage report -m" +coverage = "coverage run manage.py test --no-input && coverage report -m" report = "coverage report -m" lint = "pre-commit run --all-files" precommit = "pre-commit install" diff --git a/static-builds/README.md b/static-builds/README.md index 9b86ed08..a3c7962b 100644 --- a/static-builds/README.md +++ b/static-builds/README.md @@ -27,16 +27,29 @@ Alternatively, you can use the [Dockerfile](/Dockerfile) and extract the build. Both output their builds to a `build/` directory. ### Deploying To Netlify -To deploy to netlify, link your site GitHub repository to a netlify site, and use the following settings: +To deploy to netlify, link your site GitHub repository to a netlify site, and use the settings below. +The netlify build script uses the site API to fetch and download the artifact, using a GitHub app that +can access the repo. The app must have the `actions` and `artifacts` scopes enabled. +### Netlify Settings Build Command: -`python -m pip install httpx==0.19.0 && python static-builds/netlify_build.py` +`python -m pip install httpx==0.23.0 && python static-builds/netlify_build.py` Publish Directory: `build` -Environment Variables: -- PYTHON_VERSION: 3.8 +**Environment Variables** + +| Name | Value | Description | +|----------------|--------------------------------|-------------------------------------------------------------------------------------------| +| PYTHON_VERSION | 3.8 | The python version. Supported options are defined by netlify [here][netlify build image]. | +| API_URL | https://pythondiscord.com/ | The link to the API, which will be used to fetch the build artifacts. | +| ACTION_NAME | Build & Publish Static Preview | The name of the workflow which will be used to find the artifact. | +| ARTIFACT_NAME | static-build | The name of the artifact to download. | + + +[netlify build image]: https://github.com/netlify/build-image/tree/focal + Note that at this time, if you are deploying to netlify yourself, you won't have access to the @@ -45,6 +58,3 @@ You can either update the pack to one which will work on your domain, or you'll > Warning: If you are modifying the [build script](./netlify_build.py), make sure it is compatible with Python 3.8. - -Note: The build script uses [nightly.link](https://github.com/oprypin/nightly.link) -to fetch the artifact with no authentication. diff --git a/static-builds/netlify_build.py b/static-builds/netlify_build.py index 4e1e6106..f3a53f72 100644 --- a/static-builds/netlify_build.py +++ b/static-builds/netlify_build.py @@ -4,106 +4,55 @@ # This script performs all the actions required to build and deploy our project on netlify # It depends on the following packages, which are set in the netlify UI: -# httpx == 0.19.0 +# httpx == 0.23.0 +import json import os import time -import typing import zipfile from pathlib import Path from urllib import parse import httpx -API_URL = "https://api.github.com" -NIGHTLY_URL = "https://nightly.link" -OWNER, REPO = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] +def raise_response(response: httpx.Response) -> None: + """Raise an exception from a response if necessary.""" + if response.status_code // 100 != 2: + try: + print(response.json()) + except json.JSONDecodeError: + pass -def get_build_artifact() -> typing.Tuple[int, str]: - """ - Search for a build artifact, and return the result. + response.raise_for_status() - The return is a tuple of the check suite ID, and the URL to the artifacts. - """ - print("Fetching build URL.") - if os.getenv("PULL_REQUEST").lower() == "true": - print(f"Fetching data for PR #{os.getenv('REVIEW_ID')}") - - pull_url = f"{API_URL}/repos/{OWNER}/{REPO}/pulls/{os.getenv('REVIEW_ID')}" - pull_request = httpx.get(pull_url) - pull_request.raise_for_status() - - commit_sha = pull_request.json()["head"]["sha"] - - workflows_params = parse.urlencode({ - "event": "pull_request", - "per_page": 100 - }) - - else: - commit_sha = os.getenv("COMMIT_REF") - - workflows_params = parse.urlencode({ - "event": "push", - "per_page": 100 - }) - - print(f"Fetching action data for commit {commit_sha}") - - workflows = httpx.get(f"{API_URL}/repos/{OWNER}/{REPO}/actions/runs?{workflows_params}") - workflows.raise_for_status() - - for run in workflows.json()["workflow_runs"]: - if run["name"] == "Build & Publish Static Preview" and commit_sha == run["head_sha"]: - print(f"Found action for this commit: {run['id']}\n{run['html_url']}") - break - else: - raise Exception("Could not find the workflow run for this event.") - - polls = 0 - while polls <= 20: - if run["status"] != "completed": - print("Action isn't ready, sleeping for 10 seconds.") - polls += 1 - time.sleep(10) - - elif run["conclusion"] != "success": - print("Aborting build due to a failure in a previous CI step.") - exit(0) - - else: - print(f"Found artifact URL:\n{run['artifacts_url']}") - return run["check_suite_id"], run["artifacts_url"] - - _run = httpx.get(run["url"]) - _run.raise_for_status() - run = _run.json() - - raise Exception("Polled for the artifact workflow, but it was not ready in time.") - - -def download_artifact(suite_id: int, url: str) -> None: - """Download a build artifact from `url`, and unzip the content.""" - print("Fetching artifact data.") - - artifacts = httpx.get(url) - artifacts.raise_for_status() - artifacts = artifacts.json() - - if artifacts["total_count"] == "0": - raise Exception(f"No artifacts were found for this build, aborting.\n{url}") - - for artifact in artifacts["artifacts"]: - if artifact["name"] == "static-build": - print("Found artifact with build.") - break - else: - raise Exception("Could not find an artifact with the expected name.") - - artifact_url = f"{NIGHTLY_URL}/{OWNER}/{REPO}/suites/{suite_id}/artifacts/{artifact['id']}" - zipped_content = httpx.get(artifact_url) +if __name__ == "__main__": + owner, repo = parse.urlparse(os.getenv("REPOSITORY_URL")).path.lstrip("/").split("/")[0:2] + + download_url = "/".join([ + os.getenv("API_URL").rstrip("/"), + "api/github/artifact", + owner, + repo, + os.getenv("COMMIT_REF"), + parse.quote(os.getenv("ACTION_NAME")), + os.getenv("ARTIFACT_NAME"), + ]) + print(f"Fetching download URL from {download_url}") + response = httpx.get(download_url, follow_redirects=True) + raise_response(response) + + # The workflow is still pending, retry in a bit + while response.status_code == 202: + print(f"{response.json()['error']}. Retrying in 10 seconds.") + time.sleep(10) + response = httpx.get(download_url, follow_redirects=True) + + raise_response(response) + url = response.json()["url"] + print(f"Downloading build from {url}") + zipped_content = httpx.get(url, follow_redirects=True, timeout=3 * 60) zipped_content.raise_for_status() zip_file = Path("temp.zip") @@ -115,8 +64,3 @@ def download_artifact(suite_id: int, url: str) -> None: zip_file.unlink(missing_ok=True) print("Wrote artifact content to target directory.") - - -if __name__ == "__main__": - print("Build started") - download_artifact(*get_build_artifact()) |