diff options
| -rw-r--r-- | .github/CODEOWNERS | 11 | ||||
| -rw-r--r-- | .github/review-policy.yml | 3 | ||||
| -rw-r--r-- | .github/workflows/lint-test.yaml | 22 | ||||
| -rw-r--r-- | .github/workflows/status_embed.yaml | 78 | ||||
| -rw-r--r-- | README.md | 2 | ||||
| -rwxr-xr-x | manage.py | 6 | ||||
| -rw-r--r-- | pydis_site/apps/home/tests/test_repodata_helpers.py | 8 | ||||
| -rw-r--r-- | pydis_site/apps/home/views/home.py | 149 | ||||
| -rw-r--r-- | pydis_site/settings.py | 4 | ||||
| -rw-r--r-- | pydis_site/templates/home/index.html | 98 |
10 files changed, 243 insertions, 138 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd8eb4f3..0ba2c55b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,18 +1,15 @@ -# Request Dennis for any PR -* @Den4200 - # Infractions API pydis_site/apps/api/models/bot/infraction.py @MarkKoz pydis_site/apps/api/viewsets/bot/infraction.py @MarkKoz # Django ORM **/migrations/** @Akarys42 -**/models/** @Akarys42 +**/models/** @Akarys42 @Den4200 # CI & Docker -.github/workflows/** @MarkKoz @Akarys42 @SebastiaanZ -Dockerfile @MarkKoz @Akarys42 -docker-compose.yml @MarkKoz @Akarys42 +.github/workflows/** @MarkKoz @Akarys42 @SebastiaanZ @Den4200 +Dockerfile @MarkKoz @Akarys42 @Den4200 +docker-compose.yml @MarkKoz @Akarys42 @Den4200 # Tools Pipfile* @Akarys42 diff --git a/.github/review-policy.yml b/.github/review-policy.yml new file mode 100644 index 00000000..421b30f8 --- /dev/null +++ b/.github/review-policy.yml @@ -0,0 +1,3 @@ +remote: python-discord/.github +path: review-policies/core-developers.yml +ref: main diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 668c888d..397c2085 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -118,3 +118,25 @@ jobs: - name: Tear down docker-compose containers run: docker-compose stop if: ${{ always() }} + + # Prepare the Pull Request Payload artifact. If this fails, we + # we fail silently using the `continue-on-error` option. It's + # nice if this succeeds, but if it fails for any reason, it + # does not mean that our lint-test checks failed. + - name: Prepare Pull Request Payload artifact + id: prepare-artifact + if: always() && github.event_name == 'pull_request' + continue-on-error: true + run: cat $GITHUB_EVENT_PATH | jq '.pull_request' > pull_request_payload.json + + # This only makes sense if the previous step succeeded. To + # get the original outcome of the previous step before the + # `continue-on-error` conclusion is applied, we use the + # `.outcome` value. This step also fails silently. + - name: Upload a Build Artifact + if: always() && steps.prepare-artifact.outcome == 'success' + continue-on-error: true + uses: actions/upload-artifact@v2 + with: + name: pull-request-payload + path: pull_request_payload.json diff --git a/.github/workflows/status_embed.yaml b/.github/workflows/status_embed.yaml new file mode 100644 index 00000000..b6a71b88 --- /dev/null +++ b/.github/workflows/status_embed.yaml @@ -0,0 +1,78 @@ +name: Status Embed + +on: + workflow_run: + workflows: + - Lint & Test + - Build + - Deploy + types: + - completed + +jobs: + status_embed: + # We need to send a status embed whenever the workflow + # sequence we're running terminates. There are a number + # of situations in which that happens: + # + # 1. We reach the end of the Deploy workflow, without + # it being skipped. + # + # 2. A `pull_request` triggered a Lint & Test workflow, + # as the sequence always terminates with one run. + # + # 3. If any workflow ends in failure or was cancelled. + if: >- + (github.event.workflow_run.name == 'Deploy' && github.event.workflow_run.conclusion != 'skipped') || + github.event.workflow_run.event == 'pull_request' || + github.event.workflow_run.conclusion == 'failure' || + github.event.workflow_run.conclusion == 'cancelled' + name: Send Status Embed to Discord + runs-on: ubuntu-latest + + steps: + # A workflow_run event does not contain all the information + # we need for a PR embed. That's why we upload an artifact + # with that information in the Lint workflow. + - name: Get Pull Request Information + id: pr_info + if: github.event.workflow_run.event == 'pull_request' + run: | + curl -s -H "Authorization: token $GITHUB_TOKEN" ${{ github.event.workflow_run.artifacts_url }} > artifacts.json + DOWNLOAD_URL=$(cat artifacts.json | jq -r '.artifacts[] | select(.name == "pull-request-payload") | .archive_download_url') + [ -z "$DOWNLOAD_URL" ] && exit 1 + wget --quiet --header="Authorization: token $GITHUB_TOKEN" -O pull_request_payload.zip $DOWNLOAD_URL || exit 2 + unzip -p pull_request_payload.zip > pull_request_payload.json + [ -s pull_request_payload.json ] || exit 3 + echo "::set-output name=pr_author_login::$(jq -r '.user.login // empty' pull_request_payload.json)" + echo "::set-output name=pr_number::$(jq -r '.number // empty' pull_request_payload.json)" + echo "::set-output name=pr_title::$(jq -r '.title // empty' pull_request_payload.json)" + echo "::set-output name=pr_source::$(jq -r '.head.label // empty' pull_request_payload.json)" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Send an informational status embed to Discord instead of the + # standard embeds that Discord sends. This embed will contain + # more information and we can fine tune when we actually want + # to send an embed. + - name: GitHub Actions Status Embed for Discord + uses: SebastiaanZ/[email protected] + with: + # Our GitHub Actions webhook + webhook_id: '784184528997842985' + webhook_token: ${{ secrets.GHA_WEBHOOK_TOKEN }} + + # Workflow information + workflow_name: ${{ github.event.workflow_run.name }} + run_id: ${{ github.event.workflow_run.id }} + run_number: ${{ github.event.workflow_run.run_number }} + status: ${{ github.event.workflow_run.conclusion }} + actor: ${{ github.actor }} + repository: ${{ github.repository }} + ref: ${{ github.ref }} + sha: ${{ github.event.workflow_run.head_sha }} + + pr_author_login: ${{ steps.pr_info.outputs.pr_author_login }} + pr_number: ${{ steps.pr_info.outputs.pr_number }} + pr_title: ${{ steps.pr_info.outputs.pr_title }} + pr_source: ${{ steps.pr_info.outputs.pr_source }} @@ -1,5 +1,5 @@ # Python Discord: Site -[](https://discord.gg/2B963hn) +[](https://discord.gg/2B963hn) [![Lint & Test][1]][2] [![Build & Deploy][3]][4] [![Coverage Status][5]][6] @@ -163,7 +163,11 @@ class SiteManager: "-b", "0.0.0.0:8000", "pydis_site.wsgi:application", "--threads", "8", - "-w", "4" + "-w", "4", + "--max-requests", "1000", + "--max-requests-jitter", "50", + "--statsd-host", "graphite.default.svc.cluster.local:8125", + "--statsd-prefix", "site", ] # Run gunicorn for the production server. diff --git a/pydis_site/apps/home/tests/test_repodata_helpers.py b/pydis_site/apps/home/tests/test_repodata_helpers.py index 77b1a68d..34bbdcde 100644 --- a/pydis_site/apps/home/tests/test_repodata_helpers.py +++ b/pydis_site/apps/home/tests/test_repodata_helpers.py @@ -123,10 +123,4 @@ class TestRepositoryMetadataHelpers(TestCase): mock_get.return_value.json.return_value = ['garbage'] metadata = self.home_view._get_repo_data() - self.assertEquals(len(metadata), len(self.home_view.repos)) - for item in metadata: - with self.subTest(item=item): - self.assertEqual(item.description, "Not available.") - self.assertEqual(item.forks, 999) - self.assertEqual(item.stargazers, 999) - self.assertEqual(item.language, "Python") + self.assertEquals(len(metadata), 0) diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index c1c2055c..0e5d4edf 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -1,4 +1,4 @@ -import datetime +import logging from typing import Dict, List import requests @@ -10,11 +10,13 @@ from django.views import View from pydis_site.apps.home.models import RepositoryMetadata +log = logging.getLogger(__name__) + class HomeView(View): """The main landing page for the website.""" - github_api = "https://api.github.com/users/python-discord/repos" + github_api = "https://api.github.com/users/python-discord/repos?per_page=100" repository_cache_ttl = 3600 # Which of our GitHub repos should be displayed on the front page, and in which order? @@ -28,76 +30,88 @@ class HomeView(View): ] def _get_api_data(self) -> Dict[str, Dict[str, str]]: - """Call the GitHub API and get information about our repos.""" - repo_dict: Dict[str, dict] = {repo_name: {} for repo_name in self.repos} + """ + Call the GitHub API and get information about our repos. + + If we're unable to get that info for any reason, return an empty dict. + """ + repo_dict = {} # Fetch the data from the GitHub API api_data: List[dict] = requests.get(self.github_api).json() # Process the API data into our dict for repo in api_data: - full_name = repo["full_name"] - - if full_name in self.repos: - repo_dict[full_name] = { - "full_name": repo["full_name"], - "description": repo["description"], - "language": repo["language"], - "forks_count": repo["forks_count"], - "stargazers_count": repo["stargazers_count"], - } + try: + full_name = repo["full_name"] + + if full_name in self.repos: + repo_dict[full_name] = { + "full_name": repo["full_name"], + "description": repo["description"], + "language": repo["language"], + "forks_count": repo["forks_count"], + "stargazers_count": repo["stargazers_count"], + } + # Something is not right about the API data we got back from GitHub. + except (TypeError, ConnectionError, KeyError) as e: + log.error( + "Unable to parse the GitHub repository metadata from response!", + extra={ + 'api_data': api_data, + 'error': e + } + ) + continue return repo_dict def _get_repo_data(self) -> List[RepositoryMetadata]: """Build a list of RepositoryMetadata objects that we can use to populate the front page.""" - # Try to get site data from the cache - try: - repo_data = RepositoryMetadata.objects.get(repo_name="python-discord/site") + database_repositories = [] - # If the data is stale, we should refresh it. - if (timezone.now() - repo_data.last_updated).seconds > self.repository_cache_ttl: + # First, let's see if we have any metadata cached. + cached_data = RepositoryMetadata.objects.all() - # Try to get new data from the API. If it fails, return the cached data. - try: - api_repositories = self._get_api_data() - except (TypeError, ConnectionError): - return RepositoryMetadata.objects.all() - database_repositories = [] - - # Update or create all RepoData objects in self.repos - for repo_name, api_data in api_repositories.items(): - try: - repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) - repo_data.description = api_data["description"] - repo_data.language = api_data["language"] - repo_data.forks = api_data["forks_count"] - repo_data.stargazers = api_data["stargazers_count"] - except RepositoryMetadata.DoesNotExist: - repo_data = RepositoryMetadata( - repo_name=api_data["full_name"], - description=api_data["description"], - forks=api_data["forks_count"], - stargazers=api_data["stargazers_count"], - language=api_data["language"], - ) - repo_data.save() - database_repositories.append(repo_data) - return database_repositories - - # Otherwise, if the data is fresher than 2 minutes old, we should just return it. - else: - return RepositoryMetadata.objects.all() + # If we don't, we have to create some! + if not cached_data: - # If this is raised, the database has no repodata at all, we will create them all. - except RepositoryMetadata.DoesNotExist: - database_repositories = [] - try: - # Get new data from API - api_repositories = self._get_api_data() + # Try to get new data from the API. If it fails, we'll return an empty list. + # In this case, we simply don't display our projects on the site. + api_repositories = self._get_api_data() + + # Create all the repodata records in the database. + for api_data in api_repositories.values(): + repo_data = RepositoryMetadata( + repo_name=api_data["full_name"], + description=api_data["description"], + forks=api_data["forks_count"], + stargazers=api_data["stargazers_count"], + language=api_data["language"], + ) - # Create all the repodata records in the database. - for api_data in api_repositories.values(): + repo_data.save() + database_repositories.append(repo_data) + + return database_repositories + + # If the data is stale, we should refresh it. + if (timezone.now() - cached_data[0].last_updated).seconds > self.repository_cache_ttl: + # Try to get new data from the API. If it fails, return the cached data. + api_repositories = self._get_api_data() + + if not api_repositories: + return RepositoryMetadata.objects.all() + + # Update or create all RepoData objects in self.repos + for repo_name, api_data in api_repositories.items(): + try: + repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) + repo_data.description = api_data["description"] + repo_data.language = api_data["language"] + repo_data.forks = api_data["forks_count"] + repo_data.stargazers = api_data["stargazers_count"] + except RepositoryMetadata.DoesNotExist: repo_data = RepositoryMetadata( repo_name=api_data["full_name"], description=api_data["description"], @@ -105,23 +119,14 @@ class HomeView(View): stargazers=api_data["stargazers_count"], language=api_data["language"], ) - repo_data.save() - database_repositories.append(repo_data) - except TypeError: - for repo_name in self.repos: - repo_data = RepositoryMetadata( - last_updated=timezone.now() - datetime.timedelta(minutes=50), - repo_name=repo_name, - description="Not available.", - forks=999, - stargazers=999, - language="Python", - ) - repo_data.save() - database_repositories.append(repo_data) - + repo_data.save() + database_repositories.append(repo_data) return database_repositories + # Otherwise, if the data is fresher than 2 minutes old, we should just return it. + else: + return RepositoryMetadata.objects.all() + def get(self, request: WSGIRequest) -> HttpResponse: """Collect repo data and render the homepage view.""" repo_data = self._get_repo_data() diff --git a/pydis_site/settings.py b/pydis_site/settings.py index 204ce58f..449a343f 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -28,11 +28,11 @@ if typing.TYPE_CHECKING: env = environ.Env( DEBUG=(bool, False), - SITE_SENTRY_DSN=(str, "") + SITE_DSN=(str, "") ) sentry_sdk.init( - dsn=env('SITE_SENTRY_DSN'), + dsn=env('SITE_DSN'), integrations=[DjangoIntegration()], send_default_pii=True, release=f"pydis-site@{GIT_SHA}" diff --git a/pydis_site/templates/home/index.html b/pydis_site/templates/home/index.html index 72a5f67c..a98613a3 100644 --- a/pydis_site/templates/home/index.html +++ b/pydis_site/templates/home/index.html @@ -130,57 +130,59 @@ </section> <!-- Projects --> - <section id="projects" class="section"> - <div class="container"> - <h1 class="is-size-1">Projects</h1> - - <div class="columns is-multiline is-tablet"> - - {# Generate project data from HomeView.repos #} - {% for repo in repo_data %} - <div class="column is-one-third-desktop is-half-tablet"> - - <a href="https://github.com/{{ repo.repo_name }}"> - <article class="card"> - - <header class="card-header"> - <span class="card-header-icon"> - <span class="icon"><i class="fab fa-github"></i></span> - </span> - <div class="card-header-title"> - {{ repo.repo_name|cut:"python-discord/" }} - </div> - </header> - - <p class="card-content"> - {{ repo.description }} - </p> - - <footer class="card-footer"> - <div class="card-footer-item"> - <i class="repo-language-dot {{ repo.language | lower }}"></i> - {{ repo.language }} - </div> - <div class="card-footer-item"> - <i class="fas fa-star"></i> - {{ repo.stargazers }} - </div> - <div class="card-footer-item"> - <i class="fas fa-code-branch"></i> - {{ repo.forks }} - </div> - </footer> - - </article> - </a> + {% if repo_data %} + <section id="projects" class="section"> + <div class="container"> + <h1 class="is-size-1">Projects</h1> + + <div class="columns is-multiline is-tablet"> + + {# Generate project data from HomeView.repos #} + {% for repo in repo_data %} + <div class="column is-one-third-desktop is-half-tablet"> + + <a href="https://github.com/{{ repo.repo_name }}"> + <article class="card"> + + <header class="card-header"> + <span class="card-header-icon"> + <span class="icon"><i class="fab fa-github"></i></span> + </span> + <div class="card-header-title"> + {{ repo.repo_name|cut:"python-discord/" }} + </div> + </header> + + <p class="card-content"> + {{ repo.description }} + </p> + + <footer class="card-footer"> + <div class="card-footer-item"> + <i class="repo-language-dot {{ repo.language | lower }}"></i> + {{ repo.language }} + </div> + <div class="card-footer-item"> + <i class="fas fa-star"></i> + {{ repo.stargazers }} + </div> + <div class="card-footer-item"> + <i class="fas fa-code-branch"></i> + {{ repo.forks }} + </div> + </footer> + + </article> + </a> - </div> - {% endfor %} + </div> + {% endfor %} - </div> + </div> - </div> - </section> + </div> + </section> + {% endif %} <!-- Sponsors --> <section id="sponsors" class="hero is-light"> |