From 01a0e825333812dff21351201fce8c25031cc8fa Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 3 Sep 2020 00:39:25 +0200 Subject: Update landing page. Replace flake8-annotations with metricity. --- pydis_site/apps/home/views/home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 20e38ab0..3b5cd5ac 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -23,7 +23,7 @@ class HomeView(View): "python-discord/bot", "python-discord/snekbox", "python-discord/seasonalbot", - "python-discord/flake8-annotations", + "python-discord/metricity", "python-discord/django-simple-bulma", ] -- cgit v1.2.3 From 1b07a0021ad61f21f4414e8bb9045d605ce87f64 Mon Sep 17 00:00:00 2001 From: Derek Date: Wed, 2 Sep 2020 22:17:44 -0400 Subject: add timeline view add timeline route --- pydis_site/apps/home/urls.py | 3 ++- pydis_site/apps/home/views/__init__.py | 4 ++-- pydis_site/apps/home/views/home.py | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/urls.py b/pydis_site/apps/home/urls.py index 61e87a39..cf0c7457 100644 --- a/pydis_site/apps/home/urls.py +++ b/pydis_site/apps/home/urls.py @@ -6,7 +6,7 @@ from django.contrib.messages import ERROR from django.urls import include, path from pydis_site.utils.views import MessageRedirectView -from .views import AccountDeleteView, AccountSettingsView, HomeView +from .views import AccountDeleteView, AccountSettingsView, HomeView, timeline app_name = 'home' urlpatterns = [ @@ -38,4 +38,5 @@ urlpatterns = [ path('admin/', admin.site.urls), path('notifications/', include('django_nyt.urls')), + path('timeline/', timeline), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/pydis_site/apps/home/views/__init__.py b/pydis_site/apps/home/views/__init__.py index 801fd398..36b88b1b 100644 --- a/pydis_site/apps/home/views/__init__.py +++ b/pydis_site/apps/home/views/__init__.py @@ -1,4 +1,4 @@ from .account import DeleteView as AccountDeleteView, SettingsView as AccountSettingsView -from .home import HomeView +from .home import HomeView, timeline -__all__ = ["AccountDeleteView", "AccountSettingsView", "HomeView"] +__all__ = ["AccountDeleteView", "AccountSettingsView", "HomeView", "timeline"] diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 20e38ab0..7ad995cc 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -126,3 +126,8 @@ class HomeView(View): """Collect repo data and render the homepage view.""" repo_data = self._get_repo_data() return render(request, "home/index.html", {"repo_data": repo_data}) + + +def timeline(request: WSGIRequest) -> HttpResponse: + """Render timeline view.""" + return render(request, 'home/timeline.html') -- cgit v1.2.3 From 5f8fc4ba6395ab32866865c2a7b7c7d1c88528d3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Sat, 21 Nov 2020 00:40:34 +0100 Subject: Change seasonalbot to sir-lancebot --- pydis_site/apps/home/views/home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 09969f1d..c1c2055c 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -22,7 +22,7 @@ class HomeView(View): "python-discord/site", "python-discord/bot", "python-discord/snekbox", - "python-discord/seasonalbot", + "python-discord/sir-lancebot", "python-discord/metricity", "python-discord/django-simple-bulma", ] -- cgit v1.2.3 From bf06cc2c64b8c991070e6364c9ddc05b7da243f8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 14:06:05 +0100 Subject: Add some sanity to frontpage projects. If no API data is available, the front page shouldn't crash and burn. This commit will ensure the front page will just render without the projects it can't get API data for instead of crashing the whole thing. --- pydis_site/apps/home/views/home.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index c1c2055c..e9ed0df0 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -1,4 +1,5 @@ import datetime +import logging from typing import Dict, List import requests @@ -10,6 +11,8 @@ from django.views import View from pydis_site.apps.home.models import RepositoryMetadata +log = logging.getLogger(__name__) + class HomeView(View): """The main landing page for the website.""" @@ -67,22 +70,25 @@ class HomeView(View): # Update or create all RepoData objects in self.repos for repo_name, api_data in api_repositories.items(): - try: - repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) - repo_data.description = api_data["description"] - repo_data.language = api_data["language"] - repo_data.forks = api_data["forks_count"] - repo_data.stargazers = api_data["stargazers_count"] - except RepositoryMetadata.DoesNotExist: - repo_data = RepositoryMetadata( - repo_name=api_data["full_name"], - description=api_data["description"], - forks=api_data["forks_count"], - stargazers=api_data["stargazers_count"], - language=api_data["language"], - ) - repo_data.save() - database_repositories.append(repo_data) + if api_data: + try: + repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) + repo_data.description = api_data["description"] + repo_data.language = api_data["language"] + repo_data.forks = api_data["forks_count"] + repo_data.stargazers = api_data["stargazers_count"] + except RepositoryMetadata.DoesNotExist: + repo_data = RepositoryMetadata( + repo_name=api_data["full_name"], + description=api_data["description"], + forks=api_data["forks_count"], + stargazers=api_data["stargazers_count"], + language=api_data["language"], + ) + repo_data.save() + database_repositories.append(repo_data) + else: + log.error(f"No API data is available for {repo_name}!") return database_repositories # Otherwise, if the data is fresher than 2 minutes old, we should just return it. -- cgit v1.2.3 From d2d31b41d6122ee32fd8efebc0a3c9d33e713275 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 14:52:51 +0100 Subject: Add even more sanity for the HomeView. This rewrite will ensure we always return whichever repositories we're able to resolve, and none of the others. It also shortens the code, shrinks all the try/excepts, and generally makes things a little more readable. --- pydis_site/apps/home/views/home.py | 153 ++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 72 deletions(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index e9ed0df0..ddbfc4b4 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -1,4 +1,3 @@ -import datetime import logging from typing import Dict, List @@ -31,79 +30,59 @@ class HomeView(View): ] def _get_api_data(self) -> Dict[str, Dict[str, str]]: - """Call the GitHub API and get information about our repos.""" - repo_dict: Dict[str, dict] = {repo_name: {} for repo_name in self.repos} + """ + Call the GitHub API and get information about our repos. + + If we're unable to get that info for any reason, return an empty dict. + """ + repo_dict = {} # Fetch the data from the GitHub API api_data: List[dict] = requests.get(self.github_api).json() # Process the API data into our dict for repo in api_data: - full_name = repo["full_name"] - - if full_name in self.repos: - repo_dict[full_name] = { - "full_name": repo["full_name"], - "description": repo["description"], - "language": repo["language"], - "forks_count": repo["forks_count"], - "stargazers_count": repo["stargazers_count"], - } + try: + full_name = repo["full_name"] + + if full_name in self.repos: + repo_dict[full_name] = { + "full_name": repo["full_name"], + "description": repo["description"], + "language": repo["language"], + "forks_count": repo["forks_count"], + "stargazers_count": repo["stargazers_count"], + } + # Something is not right about the API data we got back from GitHub. + except (TypeError, ConnectionError, KeyError) as e: + log.error( + "Unable to parse the GitHub repository metadata from response!", + extra={ + 'api_data': api_data, + 'error': e + } + ) + continue return repo_dict def _get_repo_data(self) -> List[RepositoryMetadata]: """Build a list of RepositoryMetadata objects that we can use to populate the front page.""" - # Try to get site data from the cache - try: - repo_data = RepositoryMetadata.objects.get(repo_name="python-discord/site") + database_repositories = [] - # If the data is stale, we should refresh it. - if (timezone.now() - repo_data.last_updated).seconds > self.repository_cache_ttl: + # First, let's see if we have any metadata cached. + cached_data = RepositoryMetadata.objects.all() - # Try to get new data from the API. If it fails, return the cached data. - try: - api_repositories = self._get_api_data() - except (TypeError, ConnectionError): - return RepositoryMetadata.objects.all() - database_repositories = [] - - # Update or create all RepoData objects in self.repos - for repo_name, api_data in api_repositories.items(): - if api_data: - try: - repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) - repo_data.description = api_data["description"] - repo_data.language = api_data["language"] - repo_data.forks = api_data["forks_count"] - repo_data.stargazers = api_data["stargazers_count"] - except RepositoryMetadata.DoesNotExist: - repo_data = RepositoryMetadata( - repo_name=api_data["full_name"], - description=api_data["description"], - forks=api_data["forks_count"], - stargazers=api_data["stargazers_count"], - language=api_data["language"], - ) - repo_data.save() - database_repositories.append(repo_data) - else: - log.error(f"No API data is available for {repo_name}!") - return database_repositories - - # Otherwise, if the data is fresher than 2 minutes old, we should just return it. - else: - return RepositoryMetadata.objects.all() + # If we don't, we have to create some! + if not cached_data: - # If this is raised, the database has no repodata at all, we will create them all. - except RepositoryMetadata.DoesNotExist: - database_repositories = [] - try: - # Get new data from API - api_repositories = self._get_api_data() + # Try to get new data from the API. If it fails, return an empty list. + # In this case, we simply don't display our projects on the site. + api_repositories = self._get_api_data() - # Create all the repodata records in the database. - for api_data in api_repositories.values(): + # Create all the repodata records in the database. + for api_data in api_repositories.values(): + try: repo_data = RepositoryMetadata( repo_name=api_data["full_name"], description=api_data["description"], @@ -111,23 +90,53 @@ class HomeView(View): stargazers=api_data["stargazers_count"], language=api_data["language"], ) - repo_data.save() - database_repositories.append(repo_data) - except TypeError: - for repo_name in self.repos: - repo_data = RepositoryMetadata( - last_updated=timezone.now() - datetime.timedelta(minutes=50), - repo_name=repo_name, - description="Not available.", - forks=999, - stargazers=999, - language="Python", + # This error indicates there's something not quite right about the api_data types. + # In that case, just skip this repo. + except TypeError: + log.error( + "Encountered a TypeError while processing RepositoryMetadata " + "from the GitHub API.", + extra=api_data ) - repo_data.save() - database_repositories.append(repo_data) + continue + + repo_data.save() + database_repositories.append(repo_data) return database_repositories + # If the data is stale, we should refresh it. + if (timezone.now() - cached_data[0].last_updated).seconds > self.repository_cache_ttl: + # Try to get new data from the API. If it fails, return the cached data. + api_repositories = self._get_api_data() + + if not api_repositories: + return RepositoryMetadata.objects.all() + + # Update or create all RepoData objects in self.repos + for repo_name, api_data in api_repositories.items(): + try: + repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) + repo_data.description = api_data["description"] + repo_data.language = api_data["language"] + repo_data.forks = api_data["forks_count"] + repo_data.stargazers = api_data["stargazers_count"] + except RepositoryMetadata.DoesNotExist: + repo_data = RepositoryMetadata( + repo_name=api_data["full_name"], + description=api_data["description"], + forks=api_data["forks_count"], + stargazers=api_data["stargazers_count"], + language=api_data["language"], + ) + repo_data.save() + database_repositories.append(repo_data) + return database_repositories + + # Otherwise, if the data is fresher than 2 minutes old, we should just return it. + else: + return RepositoryMetadata.objects.all() + def get(self, request: WSGIRequest) -> HttpResponse: """Collect repo data and render the homepage view.""" repo_data = self._get_repo_data() -- cgit v1.2.3 From 9f5e2cd4a46349d161d5ec22c65d212a52215c47 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 15:07:03 +0100 Subject: Add a test repo with bad types. This should restore 100% coverage. --- pydis_site/apps/home/tests/mock_github_api_response.json | 7 +++++++ pydis_site/apps/home/views/home.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/tests/mock_github_api_response.json b/pydis_site/apps/home/tests/mock_github_api_response.json index ddbffed8..00f2840d 100644 --- a/pydis_site/apps/home/tests/mock_github_api_response.json +++ b/pydis_site/apps/home/tests/mock_github_api_response.json @@ -34,6 +34,13 @@ "language": "Python", "forks_count": 31 }, + { + "full_name": "python-discord/sir-lancebot", + "description": 42, + "stargazers_count": "bad types", + "language": ["not", "the", "right", "type"], + "forks_count": "31" + }, { "full_name": "python-discord/sir-lancebot", "description": "test", diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index ddbfc4b4..77496121 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -76,7 +76,7 @@ class HomeView(View): # If we don't, we have to create some! if not cached_data: - # Try to get new data from the API. If it fails, return an empty list. + # Try to get new data from the API. If it fails, we'll return an empty list. # In this case, we simply don't display our projects on the site. api_repositories = self._get_api_data() -- cgit v1.2.3 From fa9ef1d0192a372c56a172484654c4103463ea7c Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 15:25:08 +0100 Subject: Remove pointless try/except. I don't think we need to check for TypeError here, so that saves us the trouble of testing these lines to appease coverage. --- .../apps/home/tests/mock_github_api_response.json | 7 ------- pydis_site/apps/home/views/home.py | 24 +++++++--------------- 2 files changed, 7 insertions(+), 24 deletions(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/tests/mock_github_api_response.json b/pydis_site/apps/home/tests/mock_github_api_response.json index 00f2840d..ddbffed8 100644 --- a/pydis_site/apps/home/tests/mock_github_api_response.json +++ b/pydis_site/apps/home/tests/mock_github_api_response.json @@ -34,13 +34,6 @@ "language": "Python", "forks_count": 31 }, - { - "full_name": "python-discord/sir-lancebot", - "description": 42, - "stargazers_count": "bad types", - "language": ["not", "the", "right", "type"], - "forks_count": "31" - }, { "full_name": "python-discord/sir-lancebot", "description": "test", diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 77496121..97253a0c 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -82,23 +82,13 @@ class HomeView(View): # Create all the repodata records in the database. for api_data in api_repositories.values(): - try: - repo_data = RepositoryMetadata( - repo_name=api_data["full_name"], - description=api_data["description"], - forks=api_data["forks_count"], - stargazers=api_data["stargazers_count"], - language=api_data["language"], - ) - # This error indicates there's something not quite right about the api_data types. - # In that case, just skip this repo. - except TypeError: - log.error( - "Encountered a TypeError while processing RepositoryMetadata " - "from the GitHub API.", - extra=api_data - ) - continue + repo_data = RepositoryMetadata( + repo_name=api_data["full_name"], + description=api_data["description"], + forks=api_data["forks_count"], + stargazers=api_data["stargazers_count"], + language=api_data["language"], + ) repo_data.save() database_repositories.append(repo_data) -- cgit v1.2.3 From 13e09af7d50194359c2fa5575a6e6107a9f8bc9a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 17:43:45 +0100 Subject: Clean up stale metadata when HomeView starts. We don't want to keep lots of old metadata for repos we no longer need to track on the frontpage. Previously, whenever we changed the frontpage line-up, we'd be left with useless, stale data in the database. This fixes that, and adds tests. --- .../apps/home/tests/test_repodata_helpers.py | 34 ++++++++++++++++++++++ pydis_site/apps/home/views/home.py | 6 ++++ 2 files changed, 40 insertions(+) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/tests/test_repodata_helpers.py b/pydis_site/apps/home/tests/test_repodata_helpers.py index 34bbdcde..5634bc9b 100644 --- a/pydis_site/apps/home/tests/test_repodata_helpers.py +++ b/pydis_site/apps/home/tests/test_repodata_helpers.py @@ -124,3 +124,37 @@ class TestRepositoryMetadataHelpers(TestCase): metadata = self.home_view._get_repo_data() self.assertEquals(len(metadata), 0) + + def test_cleans_up_stale_metadata(self): + """Tests that we clean up stale metadata when we start the HomeView.""" + repo_data = RepositoryMetadata( + repo_name="python-discord/INVALID", + description="testrepo", + forks=42, + stargazers=42, + language="English", + last_updated=timezone.now() - timedelta(seconds=HomeView.repository_cache_ttl + 1), + ) + repo_data.save() + self.home_view.__init__() + cached_repos = RepositoryMetadata.objects.all() + cached_names = [repo.repo_name for repo in cached_repos] + + self.assertNotIn("python-discord/INVALID", cached_names) + + def test_dont_clean_up_unstale_metadata(self): + """Tests that we don't clean up good metadata when we start the HomeView.""" + repo_data = RepositoryMetadata( + repo_name="python-discord/site", + description="testrepo", + forks=42, + stargazers=42, + language="English", + last_updated=timezone.now() - timedelta(seconds=HomeView.repository_cache_ttl + 1), + ) + repo_data.save() + self.home_view.__init__() + cached_repos = RepositoryMetadata.objects.all() + cached_names = [repo.repo_name for repo in cached_repos] + + self.assertIn("python-discord/site", cached_names) diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 97253a0c..5c535b65 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -29,6 +29,12 @@ class HomeView(View): "python-discord/django-simple-bulma", ] + def __init__(self): + """Clean up stale RepositoryMetadata.""" + for cached_repo in RepositoryMetadata.objects.all(): + if cached_repo.repo_name not in self.repos: + cached_repo.delete() + def _get_api_data(self) -> Dict[str, Dict[str, str]]: """ Call the GitHub API and get information about our repos. -- cgit v1.2.3 From aaf733373e7f3dc59263df79a6a19cc87db4b27c Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 13 Dec 2020 18:58:19 +0100 Subject: Increase per_page to 100 for GitHub API call. Snekbox was being banished to page 2 and we were not iterating pages, so it was not appearing in the data we got from our call to /repos. This commit changes the request to use `per_page=100`, which will work at least until we have >100 repos in our organisation. --- pydis_site/apps/home/views/home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 97253a0c..0e5d4edf 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -16,7 +16,7 @@ log = logging.getLogger(__name__) class HomeView(View): """The main landing page for the website.""" - github_api = "https://api.github.com/users/python-discord/repos" + github_api = "https://api.github.com/users/python-discord/repos?per_page=100" repository_cache_ttl = 3600 # Which of our GitHub repos should be displayed on the front page, and in which order? -- cgit v1.2.3 From cf759514dfc6f5c8d2c010c9abc611e9e588cb85 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 26 Dec 2020 22:10:57 +0100 Subject: Improve the RepositoryMetadata cleanup code. Co-authored-by: Dennis Pham --- pydis_site/apps/home/views/home.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'pydis_site/apps/home/views/home.py') diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 5c535b65..2c1fc0d1 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -31,9 +31,7 @@ class HomeView(View): def __init__(self): """Clean up stale RepositoryMetadata.""" - for cached_repo in RepositoryMetadata.objects.all(): - if cached_repo.repo_name not in self.repos: - cached_repo.delete() + RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete() def _get_api_data(self) -> Dict[str, Dict[str, str]]: """ -- cgit v1.2.3