diff options
Diffstat (limited to 'pydis_site/apps/api')
| -rw-r--r-- | pydis_site/apps/api/github_utils.py | 209 | ||||
| -rw-r--r-- | pydis_site/apps/api/tests/test_github_utils.py | 285 | ||||
| -rw-r--r-- | pydis_site/apps/api/urls.py | 9 | ||||
| -rw-r--r-- | pydis_site/apps/api/views.py | 53 |
4 files changed, 554 insertions, 2 deletions
diff --git a/pydis_site/apps/api/github_utils.py b/pydis_site/apps/api/github_utils.py new file mode 100644 index 00000000..7d26b147 --- /dev/null +++ b/pydis_site/apps/api/github_utils.py @@ -0,0 +1,209 @@ +"""Utilities for working with the GitHub API.""" +import dataclasses +import datetime +import math +import typing + +import httpx +import jwt + +from pydis_site import settings + +MAX_RUN_TIME = datetime.timedelta(minutes=3) +"""The maximum time allowed before an action is declared timed out.""" +ISO_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" +"""The datetime string format GitHub uses.""" + + +class ArtifactProcessingError(Exception): + """Base exception for other errors related to processing a GitHub artifact.""" + + status: int + + +class UnauthorizedError(ArtifactProcessingError): + """The application does not have permission to access the requested repo.""" + + status = 401 + + +class NotFoundError(ArtifactProcessingError): + """The requested resource could not be found.""" + + status = 404 + + +class ActionFailedError(ArtifactProcessingError): + """The requested workflow did not conclude successfully.""" + + status = 400 + + +class RunTimeoutError(ArtifactProcessingError): + """The requested workflow run was not ready in time.""" + + status = 408 + + +class RunPendingError(ArtifactProcessingError): + """The requested workflow run is still pending, try again later.""" + + status = 202 + + [email protected](frozen=True) +class WorkflowRun: + """ + A workflow run from the GitHub API. + + https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run + """ + + name: str + head_sha: str + created_at: str + status: str + conclusion: str + artifacts_url: str + + @classmethod + def from_raw(cls, data: dict[str, typing.Any]): + """Create an instance using the raw data from the API, discarding unused fields.""" + return cls(**{ + key.name: data[key.name] for key in dataclasses.fields(cls) + }) + + +def generate_token() -> str: + """ + Generate a JWT token to access the GitHub API. + + The token is valid for roughly 10 minutes after generation, before the API starts + returning 401s. + + Refer to: + https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app + """ + now = datetime.datetime.now() + return jwt.encode( + { + "iat": math.floor((now - datetime.timedelta(seconds=60)).timestamp()), # Issued at + "exp": math.floor((now + datetime.timedelta(minutes=9)).timestamp()), # Expires at + "iss": settings.GITHUB_OAUTH_APP_ID, + }, + settings.GITHUB_OAUTH_KEY, + algorithm="RS256" + ) + + +def authorize(owner: str, repo: str) -> httpx.Client: + """ + Get an access token for the requested repository. + + The process is roughly: + - GET app/installations to get a list of all app installations + - POST <app_access_token> to get a token to access the given app + - GET installation/repositories and check if the requested one is part of those + """ + client = httpx.Client( + base_url=settings.GITHUB_API, + headers={"Authorization": f"bearer {generate_token()}"}, + timeout=settings.TIMEOUT_PERIOD, + ) + + try: + # Get a list of app installations we have access to + apps = client.get("app/installations") + apps.raise_for_status() + + for app in apps.json(): + # Look for an installation with the right owner + if app["account"]["login"] != owner: + continue + + # Get the repositories of the specified owner + app_token = client.post(app["access_tokens_url"]) + app_token.raise_for_status() + client.headers["Authorization"] = f"bearer {app_token.json()['token']}" + + repos = client.get("installation/repositories") + repos.raise_for_status() + + # Search for the request repository + for accessible_repo in repos.json()["repositories"]: + if accessible_repo["name"] == repo: + # We've found the correct repository, and it's accessible with the current auth + return client + + raise NotFoundError( + "Could not find the requested repository. Make sure the application can access it." + ) + + except BaseException as e: + # Close the client if we encountered an unexpected exception + client.close() + raise e + + +def check_run_status(run: WorkflowRun) -> str: + """Check if the provided run has been completed, otherwise raise an exception.""" + created_at = datetime.datetime.strptime(run.created_at, ISO_FORMAT_STRING) + run_time = datetime.datetime.utcnow() - created_at + + if run.status != "completed": + if run_time <= MAX_RUN_TIME: + raise RunPendingError( + f"The requested run is still pending. It was created " + f"{run_time.seconds // 60}:{run_time.seconds % 60 :>02} minutes ago." + ) + else: + raise RunTimeoutError("The requested workflow was not ready in time.") + + if run.conclusion != "success": + # The action failed, or did not run + raise ActionFailedError(f"The requested workflow ended with: {run.conclusion}") + + # The requested action is ready + return run.artifacts_url + + +def get_artifact(owner: str, repo: str, sha: str, action_name: str, artifact_name: str) -> str: + """Get a download URL for a build artifact.""" + client = authorize(owner, repo) + + try: + # Get the workflow runs for this repository + runs = client.get(f"/repos/{owner}/{repo}/actions/runs", params={"per_page": 100}) + runs.raise_for_status() + runs = runs.json() + + # Filter the runs for the one associated with the given SHA + for run in runs["workflow_runs"]: + run = WorkflowRun.from_raw(run) + if run.name == action_name and sha == run.head_sha: + break + else: + raise NotFoundError( + "Could not find a run matching the provided settings in the previous hundred runs." + ) + + # Check the workflow status + url = check_run_status(run) + + # Filter the artifacts, and return the download URL + artifacts = client.get(url) + artifacts.raise_for_status() + + for artifact in artifacts.json()["artifacts"]: + if artifact["name"] == artifact_name: + data = client.get(artifact["archive_download_url"]) + if data.status_code == 302: + return str(data.next_request.url) + + # The following line is left untested since it should in theory be impossible + data.raise_for_status() # pragma: no cover + + raise NotFoundError("Could not find an artifact matching the provided name.") + + finally: + client.close() diff --git a/pydis_site/apps/api/tests/test_github_utils.py b/pydis_site/apps/api/tests/test_github_utils.py new file mode 100644 index 00000000..f642f689 --- /dev/null +++ b/pydis_site/apps/api/tests/test_github_utils.py @@ -0,0 +1,285 @@ +import dataclasses +import datetime +import typing +import unittest +from unittest import mock + +import django.test +import httpx +import jwt +import rest_framework.response +import rest_framework.test +from django.urls import reverse + +from .. import github_utils + + +class GeneralUtilityTests(unittest.TestCase): + """Test the utility methods which do not fit in another class.""" + + def test_token_generation(self): + """Test that the a valid JWT token is generated.""" + def encode(payload: dict, _: str, algorithm: str, *args, **kwargs) -> str: + """ + Intercept the encode method. + + The result is encoded with an algorithm which does not require a PEM key, as it may + not be available in testing environments. + """ + self.assertEqual("RS256", algorithm, "The GitHub App JWT must be signed using RS256.") + return original_encode( + payload, "secret-encoding-key", algorithm="HS256", *args, **kwargs + ) + + original_encode = jwt.encode + with mock.patch("jwt.encode", new=encode): + token = github_utils.generate_token() + decoded = jwt.decode(token, "secret-encoding-key", algorithms=["HS256"]) + + delta = datetime.timedelta(minutes=10) + self.assertAlmostEqual(decoded["exp"] - decoded["iat"], delta.total_seconds()) + self.assertLess(decoded["exp"], (datetime.datetime.now() + delta).timestamp()) + + +class CheckRunTests(unittest.TestCase): + """Tests the check_run_status utility.""" + + run_kwargs: typing.Mapping = { + "name": "run_name", + "head_sha": "sha", + "status": "completed", + "conclusion": "success", + "created_at": datetime.datetime.utcnow().strftime(github_utils.ISO_FORMAT_STRING), + "artifacts_url": "url", + } + + def test_completed_run(self): + """Test that an already completed run returns the correct URL.""" + final_url = "some_url_string_1234" + + kwargs = dict(self.run_kwargs, artifacts_url=final_url) + result = github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + self.assertEqual(final_url, result) + + def test_pending_run(self): + """Test that a pending run raises the proper exception.""" + kwargs = dict(self.run_kwargs, status="pending") + with self.assertRaises(github_utils.RunPendingError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + def test_timeout_error(self): + """Test that a timeout is declared after a certain duration.""" + kwargs = dict(self.run_kwargs, status="pending") + # Set the creation time to well before the MAX_RUN_TIME + # to guarantee the right conclusion + kwargs["created_at"] = ( + datetime.datetime.utcnow() - github_utils.MAX_RUN_TIME - datetime.timedelta(minutes=10) + ).strftime(github_utils.ISO_FORMAT_STRING) + + with self.assertRaises(github_utils.RunTimeoutError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + def test_failed_run(self): + """Test that a failed run raises the proper exception.""" + kwargs = dict(self.run_kwargs, conclusion="failed") + with self.assertRaises(github_utils.ActionFailedError): + github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) + + +def get_response_authorize(_: httpx.Client, request: httpx.Request, **__) -> httpx.Response: + """ + Helper method for the authorize tests. + + Requests are intercepted before being sent out, and the appropriate responses are returned. + """ + path = request.url.path + auth = request.headers.get("Authorization") + + if request.method == "GET": + if path == "/app/installations": + if auth == "bearer JWT initial token": + return httpx.Response(200, request=request, json=[{ + "account": {"login": "VALID_OWNER"}, + "access_tokens_url": "https://example.com/ACCESS_TOKEN_URL" + }]) + else: + return httpx.Response( + 401, json={"error": "auth app/installations"}, request=request + ) + + elif path == "/installation/repositories": + if auth == "bearer app access token": + return httpx.Response(200, request=request, json={ + "repositories": [{ + "name": "VALID_REPO" + }] + }) + else: # pragma: no cover + return httpx.Response( + 401, json={"error": "auth installation/repositories"}, request=request + ) + + elif request.method == "POST": + if path == "/ACCESS_TOKEN_URL": + if auth == "bearer JWT initial token": + return httpx.Response(200, request=request, json={"token": "app access token"}) + else: # pragma: no cover + return httpx.Response(401, json={"error": "auth access_token"}, request=request) + + # Reaching this point means something has gone wrong + return httpx.Response(500, request=request) # pragma: no cover + + [email protected]("httpx.Client.send", new=get_response_authorize) [email protected](github_utils, "generate_token", new=mock.Mock(return_value="JWT initial token")) +class AuthorizeTests(unittest.TestCase): + """Test the authorize utility.""" + + def test_invalid_apps_auth(self): + """Test that an exception is raised if authorization was attempted with an invalid token.""" + with mock.patch.object(github_utils, "generate_token", return_value="Invalid token"): + with self.assertRaises(httpx.HTTPStatusError) as error: + github_utils.authorize("VALID_OWNER", "VALID_REPO") + + exception: httpx.HTTPStatusError = error.exception + self.assertEqual(401, exception.response.status_code) + self.assertEqual("auth app/installations", exception.response.json()["error"]) + + def test_missing_repo(self): + """Test that an exception is raised when the selected owner or repo are not available.""" + with self.assertRaises(github_utils.NotFoundError): + github_utils.authorize("INVALID_OWNER", "VALID_REPO") + with self.assertRaises(github_utils.NotFoundError): + github_utils.authorize("VALID_OWNER", "INVALID_REPO") + + def test_valid_authorization(self): + """Test that an accessible repository can be accessed.""" + client = github_utils.authorize("VALID_OWNER", "VALID_REPO") + self.assertEqual("bearer app access token", client.headers.get("Authorization")) + + +class ArtifactFetcherTests(unittest.TestCase): + """Test the get_artifact utility.""" + + @staticmethod + def get_response_get_artifact(request: httpx.Request, **_) -> httpx.Response: + """ + Helper method for the get_artifact tests. + + Requests are intercepted before being sent out, and the appropriate responses are returned. + """ + path = request.url.path + + if "force_error" in path: + return httpx.Response(404, request=request) + + if request.method == "GET": + if path == "/repos/owner/repo/actions/runs": + run = github_utils.WorkflowRun( + name="action_name", + head_sha="action_sha", + created_at=datetime.datetime.now().strftime(github_utils.ISO_FORMAT_STRING), + status="completed", + conclusion="success", + artifacts_url="artifacts_url" + ) + return httpx.Response( + 200, request=request, json={"workflow_runs": [dataclasses.asdict(run)]} + ) + elif path == "/artifact_url": + return httpx.Response( + 200, request=request, json={"artifacts": [{ + "name": "artifact_name", + "archive_download_url": "artifact_download_url" + }]} + ) + elif path == "/artifact_download_url": + response = httpx.Response(302, request=request) + response.next_request = httpx.Request( + "GET", + httpx.URL("https://final_download.url") + ) + return response + + # Reaching this point means something has gone wrong + return httpx.Response(500, request=request) # pragma: no cover + + def setUp(self) -> None: + self.call_args = ["owner", "repo", "action_sha", "action_name", "artifact_name"] + self.client = httpx.Client(base_url="https://example.com") + + self.patchers = [ + mock.patch.object(self.client, "send", new=self.get_response_get_artifact), + mock.patch.object(github_utils, "authorize", return_value=self.client), + mock.patch.object(github_utils, "check_run_status", return_value="artifact_url"), + ] + + for patcher in self.patchers: + patcher.start() + + def tearDown(self) -> None: + for patcher in self.patchers: + patcher.stop() + + def test_client_closed_on_errors(self): + """Test that the client is terminated even if an error occurs at some point.""" + self.call_args[0] = "force_error" + with self.assertRaises(httpx.HTTPStatusError): + github_utils.get_artifact(*self.call_args) + self.assertTrue(self.client.is_closed) + + def test_missing(self): + """Test that an exception is raised if the requested artifact was not found.""" + cases = ( + "invalid sha", + "invalid action name", + "invalid artifact name", + ) + for i, name in enumerate(cases, 2): + with self.subTest(f"Test {name} raises an error"): + new_args = self.call_args.copy() + new_args[i] = name + + with self.assertRaises(github_utils.NotFoundError): + github_utils.get_artifact(*new_args) + + def test_valid(self): + """Test that the correct download URL is returned for valid requests.""" + url = github_utils.get_artifact(*self.call_args) + self.assertEqual("https://final_download.url", url) + self.assertTrue(self.client.is_closed) + + [email protected](github_utils, "get_artifact") +class GitHubArtifactViewTests(django.test.TestCase): + """Test the GitHub artifact fetch API view.""" + + def setUp(self): + self.kwargs = { + "owner": "test_owner", + "repo": "test_repo", + "sha": "test_sha", + "action_name": "test_action", + "artifact_name": "test_artifact", + } + self.url = reverse("api:github-artifacts", kwargs=self.kwargs) + + def test_correct_artifact(self, artifact_mock: mock.Mock): + """Test a proper response is returned with proper input.""" + artifact_mock.return_value = "final download url" + result = self.client.get(self.url) + + self.assertIsInstance(result, rest_framework.response.Response) + self.assertEqual({"url": artifact_mock.return_value}, result.data) + + def test_failed_fetch(self, artifact_mock: mock.Mock): + """Test that a proper error is returned when the request fails.""" + artifact_mock.side_effect = github_utils.NotFoundError("Test error message") + result = self.client.get(self.url) + + self.assertIsInstance(result, rest_framework.response.Response) + self.assertEqual({ + "error_type": github_utils.NotFoundError.__name__, + "error": "Test error message", + "requested_resource": "/".join(self.kwargs.values()) + }, result.data) diff --git a/pydis_site/apps/api/urls.py b/pydis_site/apps/api/urls.py index 1e564b29..2757f176 100644 --- a/pydis_site/apps/api/urls.py +++ b/pydis_site/apps/api/urls.py @@ -1,7 +1,7 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter -from .views import HealthcheckView, RulesView +from .views import GitHubArtifactsView, HealthcheckView, RulesView from .viewsets import ( AocAccountLinkViewSet, AocCompletionistBlockViewSet, @@ -86,5 +86,10 @@ urlpatterns = ( # from django_hosts.resolvers import reverse path('bot/', include((bot_router.urls, 'api'), namespace='bot')), path('healthcheck', HealthcheckView.as_view(), name='healthcheck'), - path('rules', RulesView.as_view(), name='rules') + path('rules', RulesView.as_view(), name='rules'), + path( + 'github/artifact/<str:owner>/<str:repo>/<str:sha>/<str:action_name>/<str:artifact_name>', + GitHubArtifactsView.as_view(), + name="github-artifacts" + ), ) diff --git a/pydis_site/apps/api/views.py b/pydis_site/apps/api/views.py index 816463f6..ad2d948e 100644 --- a/pydis_site/apps/api/views.py +++ b/pydis_site/apps/api/views.py @@ -1,7 +1,10 @@ from rest_framework.exceptions import ParseError +from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from . import github_utils + class HealthcheckView(APIView): """ @@ -152,3 +155,53 @@ class RulesView(APIView): "Do not offer or ask for paid work of any kind." ), ]) + + +class GitHubArtifactsView(APIView): + """ + Provides utilities for interacting with the GitHub API and obtaining action artifacts. + + ## Routes + ### GET /github/artifacts + Returns a download URL for the artifact requested. + + { + 'url': 'https://pipelines.actions.githubusercontent.com/...' + } + + ### Exceptions + In case of an error, the following body will be returned: + + { + "error_type": "<error class name>", + "error": "<error description>", + "requested_resource": "<owner>/<repo>/<sha>/<artifact_name>" + } + + ## Authentication + Does not require any authentication nor permissions. + """ + + authentication_classes = () + permission_classes = () + + def get( + self, + request: Request, + *, + owner: str, + repo: str, + sha: str, + action_name: str, + artifact_name: str + ) -> Response: + """Return a download URL for the requested artifact.""" + try: + url = github_utils.get_artifact(owner, repo, sha, action_name, artifact_name) + return Response({"url": url}) + except github_utils.ArtifactProcessingError as e: + return Response({ + "error_type": e.__class__.__name__, + "error": str(e), + "requested_resource": f"{owner}/{repo}/{sha}/{action_name}/{artifact_name}" + }, status=e.status) |