From cf8ce93e34ab543fdc9e4df83aa4583128c2ddc0 Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Mon, 21 Mar 2022 18:42:04 -0400 Subject: add token safety pin --- .../guides/python-guides/keeping-tokens-safe.md | 21 +++++++++++++++++++++ .../resources/guides/python-guides/token-reset.png | Bin 0 -> 132625 bytes 2 files changed, 21 insertions(+) create mode 100644 pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md create mode 100644 pydis_site/apps/content/resources/guides/python-guides/token-reset.png diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md new file mode 100644 index 00000000..8e283d70 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -0,0 +1,21 @@ +--- +title: Keeping Discord Bot Tokens Safe +description: How to keep your bot tokens safe and safety measures you can take. +--- +It's **very** important to keep a bot token safe, primarily because anyone who has the bot token can do whatever they want with the bot -- such as destroying servers your bot has been added to and getting your bot banned from the API. + +# How to Avoid Leaking your Token +To help prevent leaking your token, you should ensure that you don't upload it to an open source program/website, such as replit and github, as they show your code publicly. The best practice for storing tokens is generally utilising .env files ([click here](https://vcokltfre.dev/tips/tokens/.) for more information on storing tokens safely) + +# What should I do if my token does get leaked? + +If for whatever reason your token gets leaked, you should immediately follow these steps: +- Go to the list of [Discord Bot Applications](https://discord.com/developers/applications) you have and select the bot application that had the token leaked. +- Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) +- Press the Regenerate (2) option to regen your bot token. +![token_reset.png](token-reset.png) + +Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. + +# Summary +Make sure you keep your token secure by storing it safely, not sending it to anyone you don't trust, and regenerating your token if it does get leaked. \ No newline at end of file diff --git a/pydis_site/apps/content/resources/guides/python-guides/token-reset.png b/pydis_site/apps/content/resources/guides/python-guides/token-reset.png new file mode 100644 index 00000000..bd672b93 Binary files /dev/null and b/pydis_site/apps/content/resources/guides/python-guides/token-reset.png differ -- cgit v1.2.3 From 9ba4891e6e47f7f498eafa61e4a1d5e301f2426a Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Mon, 21 Mar 2022 18:59:26 -0400 Subject: fixing image not appearing --- .../guides/python-guides/keeping-tokens-safe.md | 3 ++- .../resources/guides/python-guides/token-reset.png | Bin 132625 -> 0 bytes 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 pydis_site/apps/content/resources/guides/python-guides/token-reset.png diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index 8e283d70..e37039d1 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -13,7 +13,8 @@ If for whatever reason your token gets leaked, you should immediately follow the - Go to the list of [Discord Bot Applications](https://discord.com/developers/applications) you have and select the bot application that had the token leaked. - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) - Press the Regenerate (2) option to regen your bot token. -![token_reset.png](token-reset.png) + +![Steps to Take to Reset your Discord Bot](https://cdn.discordapp.com/attachments/343944376055103488/845290595793764392/regen_token.png) Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. diff --git a/pydis_site/apps/content/resources/guides/python-guides/token-reset.png b/pydis_site/apps/content/resources/guides/python-guides/token-reset.png deleted file mode 100644 index bd672b93..00000000 Binary files a/pydis_site/apps/content/resources/guides/python-guides/token-reset.png and /dev/null differ -- cgit v1.2.3 From 306a906c850324a4a31a62d82d5348a18f3f2e5e Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Sat, 2 Apr 2022 12:16:13 -0400 Subject: Update pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md Co-authored-by: Xithrius <15021300+Xithrius@users.noreply.github.com> --- .../apps/content/resources/guides/python-guides/keeping-tokens-safe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index e37039d1..6b5dae34 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -12,7 +12,7 @@ To help prevent leaking your token, you should ensure that you don't upload it t If for whatever reason your token gets leaked, you should immediately follow these steps: - Go to the list of [Discord Bot Applications](https://discord.com/developers/applications) you have and select the bot application that had the token leaked. - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) -- Press the Regenerate (2) option to regen your bot token. +- Press the Regenerate option to regen your bot token. ![Steps to Take to Reset your Discord Bot](https://cdn.discordapp.com/attachments/343944376055103488/845290595793764392/regen_token.png) -- cgit v1.2.3 From 27baeb887cf14fb641635379bc80cd917a0f0a2d Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Wed, 20 Apr 2022 20:58:06 -0400 Subject: change image to accurately reflect current system --- .../apps/content/resources/guides/python-guides/keeping-tokens-safe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index e37039d1..f85b02d8 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -14,7 +14,7 @@ If for whatever reason your token gets leaked, you should immediately follow the - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) - Press the Regenerate (2) option to regen your bot token. -![Steps to Take to Reset your Discord Bot](https://cdn.discordapp.com/attachments/343944376055103488/845290595793764392/regen_token.png) +![Steps to Take to Reset your Discord Bot](https://cdn.discordapp.com/attachments/859123972884922418/966502706258784366/unknown.png) Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. -- cgit v1.2.3 From 828c7cb512fb859f577870abe2fdeaf070e4e2fb Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Wed, 20 Apr 2022 21:06:10 -0400 Subject: used wrong image --- .../apps/content/resources/guides/python-guides/keeping-tokens-safe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index 272ad45a..1789bdf1 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -14,7 +14,7 @@ If for whatever reason your token gets leaked, you should immediately follow the - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) - Press the Regenerate option to regen your bot token. -![Steps to Take to Reset your Discord Bot](https://cdn.discordapp.com/attachments/859123972884922418/966502706258784366/unknown.png) +![Steps to Take to Reset your Discord Bot](https://media.discordapp.net/attachments/859123972884922418/966504639421894697/bot_application.jpg?width=1348&height=671) Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. -- cgit v1.2.3 From b7cef2771a784615c1b6e7f0fba9545e7f17e881 Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Sat, 23 Apr 2022 12:35:34 -0400 Subject: Update pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md Co-authored-by: Bluenix --- .../apps/content/resources/guides/python-guides/keeping-tokens-safe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index 1789bdf1..3eac783e 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -12,7 +12,7 @@ To help prevent leaking your token, you should ensure that you don't upload it t If for whatever reason your token gets leaked, you should immediately follow these steps: - Go to the list of [Discord Bot Applications](https://discord.com/developers/applications) you have and select the bot application that had the token leaked. - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) -- Press the Regenerate option to regen your bot token. +- Press the Regenerate button to regenerate your bot token and invalidate the old one. ![Steps to Take to Reset your Discord Bot](https://media.discordapp.net/attachments/859123972884922418/966504639421894697/bot_application.jpg?width=1348&height=671) -- cgit v1.2.3 From c14887951f50700d009b273c3530ae517f6e820a Mon Sep 17 00:00:00 2001 From: Shom770 <82843611+Shom770@users.noreply.github.com> Date: Sat, 23 Apr 2022 12:49:51 -0400 Subject: bluenix's changes --- .../guides/python-guides/keeping-tokens-safe.md | 13 ++++++++++--- .../static/images/content/regenerating_token.jpg | Bin 0 -> 180570 bytes 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 pydis_site/static/images/content/regenerating_token.jpg diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md index 3eac783e..8e9f7075 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -2,10 +2,16 @@ title: Keeping Discord Bot Tokens Safe description: How to keep your bot tokens safe and safety measures you can take. --- -It's **very** important to keep a bot token safe, primarily because anyone who has the bot token can do whatever they want with the bot -- such as destroying servers your bot has been added to and getting your bot banned from the API. +It's **very** important to keep a bot token safe, +primarily because anyone who has the bot token can do whatever they want with the bot -- +such as destroying servers your bot has been added to and getting your bot banned from the API. # How to Avoid Leaking your Token -To help prevent leaking your token, you should ensure that you don't upload it to an open source program/website, such as replit and github, as they show your code publicly. The best practice for storing tokens is generally utilising .env files ([click here](https://vcokltfre.dev/tips/tokens/.) for more information on storing tokens safely) +To help prevent leaking your token, +you should ensure that you don't upload it to an open source program/website, +such as replit and github, as they show your code publicly. +The best practice for storing tokens is generally utilising .env files +([click here](https://vcokltfre.dev/tips/tokens/.) for more information on storing tokens safely). # What should I do if my token does get leaked? @@ -14,9 +20,10 @@ If for whatever reason your token gets leaked, you should immediately follow the - Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) - Press the Regenerate button to regenerate your bot token and invalidate the old one. -![Steps to Take to Reset your Discord Bot](https://media.discordapp.net/attachments/859123972884922418/966504639421894697/bot_application.jpg?width=1348&height=671) +![Steps to Take to Reset your Discord Bot](/static/images/content/regenerating_token.jpg) Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. +The old token will stop working though, so make sure to replace the old token with the new one in your code if you haven't already. # Summary Make sure you keep your token secure by storing it safely, not sending it to anyone you don't trust, and regenerating your token if it does get leaked. \ No newline at end of file diff --git a/pydis_site/static/images/content/regenerating_token.jpg b/pydis_site/static/images/content/regenerating_token.jpg new file mode 100644 index 00000000..7b2588dc Binary files /dev/null and b/pydis_site/static/images/content/regenerating_token.jpg differ -- cgit v1.2.3 From edcee093449fce833116ae3dbb15d6ccf3797652 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 01:53:19 +0200 Subject: Add Setting For Static Builds Adds an explicit setting for static builds instead of relying on the environment variable. Signed-off-by: Hassan Abouelela --- pydis_site/apps/home/views/home.py | 8 +++----- pydis_site/apps/redirect/urls.py | 2 +- pydis_site/settings.py | 8 +++++--- pydis_site/urls.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 9bb1f8fd..8a165682 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -32,9 +32,7 @@ class HomeView(View): def __init__(self): """Clean up stale RepositoryMetadata.""" - self._static_build = settings.env("STATIC_BUILD") - - if not self._static_build: + if not settings.STATIC_BUILD: RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete() # If no token is defined (for example in local development), then @@ -94,7 +92,7 @@ class HomeView(View): def _get_repo_data(self) -> List[RepositoryMetadata]: """Build a list of RepositoryMetadata objects that we can use to populate the front page.""" # First off, load the timestamp of the least recently updated entry. - if self._static_build: + if settings.STATIC_BUILD: last_update = None else: last_update = ( @@ -121,7 +119,7 @@ class HomeView(View): for api_data in api_repositories.values() ] - if settings.env("STATIC_BUILD"): + if settings.STATIC_BUILD: return data else: return RepositoryMetadata.objects.bulk_create(data) diff --git a/pydis_site/apps/redirect/urls.py b/pydis_site/apps/redirect/urls.py index f7ddf45b..ed1564b0 100644 --- a/pydis_site/apps/redirect/urls.py +++ b/pydis_site/apps/redirect/urls.py @@ -31,7 +31,7 @@ class Redirect: def map_redirect(name: str, data: Redirect) -> list[URLPattern]: """Return a pattern using the Redirects app, or a static HTML redirect for static builds.""" - if not settings.env("STATIC_BUILD"): + if not settings.STATIC_BUILD: # Normal dynamic redirect return [path( data.original_path, diff --git a/pydis_site/settings.py b/pydis_site/settings.py index bbf1d3aa..315ea737 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -95,11 +95,13 @@ else: ) SECRET_KEY = env('SECRET_KEY') +STATIC_BUILD: bool = env("STATIC_BUILD") + # Application definition NON_STATIC_APPS = [ 'pydis_site.apps.api', 'pydis_site.apps.staff', -] if not env("STATIC_BUILD") else [] +] if not STATIC_BUILD else [] INSTALLED_APPS = [ *NON_STATIC_APPS, @@ -130,7 +132,7 @@ if not env("BUILDING_DOCKER"): NON_STATIC_MIDDLEWARE = [ 'django_prometheus.middleware.PrometheusBeforeMiddleware', -] if not env("STATIC_BUILD") else [] +] if not STATIC_BUILD else [] # Ensure that Prometheus middlewares are first and last here. MIDDLEWARE = [ @@ -175,7 +177,7 @@ WSGI_APPLICATION = 'pydis_site.wsgi.application' DATABASES = { 'default': env.db(), 'metricity': env.db('METRICITY_DB_URL'), -} if not env("STATIC_BUILD") else {} +} if not STATIC_BUILD else {} # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators diff --git a/pydis_site/urls.py b/pydis_site/urls.py index 6cd31f26..0f2f6aeb 100644 --- a/pydis_site/urls.py +++ b/pydis_site/urls.py @@ -12,7 +12,7 @@ NON_STATIC_PATTERNS = [ path('pydis-api/', include('pydis_site.apps.api.urls', namespace='internal_api')), path('', include('django_prometheus.urls')), -] if not settings.env("STATIC_BUILD") else [] +] if not settings.STATIC_BUILD else [] urlpatterns = ( @@ -29,7 +29,7 @@ urlpatterns = ( ) -if not settings.env("STATIC_BUILD"): +if not settings.STATIC_BUILD: urlpatterns += ( path('staff/', include('pydis_site.apps.staff.urls', namespace='staff')), ) -- cgit v1.2.3 From d50028d6b92909a39139007f0f3bcd7c90a88420 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 06:08:22 +0200 Subject: Add Tags To Content Listings Adds bot tags to the content page, as well as a model to go along with it. Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/migrations/0001_initial.py | 23 ++++ pydis_site/apps/content/migrations/__init__.py | 0 pydis_site/apps/content/models/__init__.py | 0 pydis_site/apps/content/models/tag.py | 17 +++ pydis_site/apps/content/resources/tags/_info.yml | 3 + pydis_site/apps/content/utils.py | 125 ++++++++++++++++++++- pydis_site/apps/content/views/page_category.py | 9 +- pydis_site/templates/content/listing.html | 10 +- pydis_site/templates/content/page.html | 2 +- 9 files changed, 177 insertions(+), 12 deletions(-) create mode 100644 pydis_site/apps/content/migrations/0001_initial.py create mode 100644 pydis_site/apps/content/migrations/__init__.py create mode 100644 pydis_site/apps/content/models/__init__.py create mode 100644 pydis_site/apps/content/models/tag.py create mode 100644 pydis_site/apps/content/resources/tags/_info.yml diff --git a/pydis_site/apps/content/migrations/0001_initial.py b/pydis_site/apps/content/migrations/0001_initial.py new file mode 100644 index 00000000..15e3fc95 --- /dev/null +++ b/pydis_site/apps/content/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# Generated by Django 4.0.6 on 2022-08-13 00:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Tag', + fields=[ + ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), + ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), + ('body', models.TextField(help_text='The content of the tag.')), + ('url', models.URLField(help_text='The URL to this tag on GitHub.')), + ], + ), + ] diff --git a/pydis_site/apps/content/migrations/__init__.py b/pydis_site/apps/content/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py new file mode 100644 index 00000000..1437b96a --- /dev/null +++ b/pydis_site/apps/content/models/tag.py @@ -0,0 +1,17 @@ +from django.db import models + + +class Tag(models.Model): + """A tag from the python-discord server.""" + + last_updated = models.DateTimeField( + help_text="The date and time this data was last fetched.", + auto_now=True, + ) + name = models.CharField( + help_text="The tag's name.", + primary_key=True, + max_length=50, + ) + body = models.TextField(help_text="The content of the tag.") + url = models.URLField(help_text="The URL to this tag on GitHub.") diff --git a/pydis_site/apps/content/resources/tags/_info.yml b/pydis_site/apps/content/resources/tags/_info.yml new file mode 100644 index 00000000..054125ec --- /dev/null +++ b/pydis_site/apps/content/resources/tags/_info.yml @@ -0,0 +1,3 @@ +title: Tags +description: Useful snippets that are often used in the server. +icon: fas fa-tags diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index d3f270ff..a4252284 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -1,14 +1,26 @@ +import datetime +import functools +import tarfile +import tempfile +from io import BytesIO from pathlib import Path -from typing import Dict, Tuple import frontmatter +import httpx import markdown import yaml from django.http import Http404 +from django.utils import timezone from markdown.extensions.toc import TocExtension +from pydis_site import settings +from .models.tag import Tag -def get_category(path: Path) -> Dict[str, str]: +TAG_URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" +TAG_CACHE_TTL = datetime.timedelta(hours=1) + + +def get_category(path: Path) -> dict[str, str]: """Load category information by name from _info.yml.""" if not path.is_dir(): raise Http404("Category not found.") @@ -16,7 +28,7 @@ def get_category(path: Path) -> Dict[str, str]: return yaml.safe_load(path.joinpath("_info.yml").read_text(encoding="utf-8")) -def get_categories(path: Path) -> Dict[str, Dict]: +def get_categories(path: Path) -> dict[str, dict]: """Get information for all categories.""" categories = {} @@ -27,8 +39,111 @@ def get_categories(path: Path) -> Dict[str, Dict]: return categories -def get_category_pages(path: Path) -> Dict[str, Dict]: +@functools.cache +def get_tags_static() -> list[Tag]: + """ + Fetch tag information in static builds. + + This will return a cached value, so it should only be used for static builds. + """ + return fetch_tags() + + +def fetch_tags() -> list[Tag]: + """ + Fetch tag data from the GitHub API. + + The entire repository is downloaded and extracted locally because + getting file content would require one request per file, and can get rate-limited. + """ + if settings.GITHUB_TOKEN: + headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} + else: + headers = {} + + tar_file = httpx.get( + f"{settings.GITHUB_API}/repos/python-discord/bot/tarball", + follow_redirects=True, + timeout=settings.TIMEOUT_PERIOD, + headers=headers, + ) + tar_file.raise_for_status() + + tags = [] + with tempfile.TemporaryDirectory() as folder: + with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo: + included = [] + for file in repo.getmembers(): + if "/bot/resources/tags" in file.path: + included.append(file) + repo.extractall(folder, included) + + for tag_file in Path(folder).rglob("*.md"): + tags.append(Tag( + name=tag_file.name.removesuffix(".md"), + body=tag_file.read_text(encoding="utf-8"), + url=f"{TAG_URL_BASE}/{tag_file.name}" + )) + + return tags + + +def get_tags() -> list[Tag]: + """Return a list of all tags visible to the application, from the cache or API.""" + if settings.STATIC_BUILD: + last_update = None + else: + last_update = ( + Tag.objects.values_list("last_updated", flat=True) + .order_by("last_updated").first() + ) + + if last_update is None or timezone.now() >= (last_update + TAG_CACHE_TTL): + # Stale or empty cache + if settings.STATIC_BUILD: + tags = get_tags_static() + else: + tags = fetch_tags() + Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete() + for tag in tags: + tag.save() + + return tags + else: + # Get tags from database + return Tag.objects.all() + + +def get_tag(name: str) -> Tag: + """Return a tag by name.""" + tags = get_tags() + for tag in tags: + if tag.name == name: + return tag + + raise Tag.DoesNotExist() + + +def get_category_pages(path: Path) -> dict[str, dict]: """Get all page names and their metadata at a category path.""" + # Special handling for tags + if path == Path(__file__).parent / "resources/tags": + tags = {} + for tag in get_tags(): + content = frontmatter.parse(tag.body)[1] + if len(content) > 100: + # Trim the preview to a maximum of 100 visible characters + # This causes some markdown to break, but we ignore that + content = content[:100] + "..." + + tags[tag.name] = { + "title": tag.name, + "description": markdown.markdown(content), + "icon": "fas fa-tag" + } + + return {name: tags[name] for name in sorted(tags)} + pages = {} for item in path.glob("*.md"): @@ -39,7 +154,7 @@ def get_category_pages(path: Path) -> Dict[str, Dict]: return pages -def get_page(path: Path) -> Tuple[str, Dict]: +def get_page(path: Path) -> tuple[str, dict]: """Get one specific page.""" if not path.is_file(): raise Http404("Page not found.") diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py index 356eb021..01ce8402 100644 --- a/pydis_site/apps/content/views/page_category.py +++ b/pydis_site/apps/content/views/page_category.py @@ -1,4 +1,3 @@ -import typing as t from pathlib import Path import frontmatter @@ -25,7 +24,7 @@ class PageOrCategoryView(TemplateView): return super().dispatch(request, *args, **kwargs) - def get_template_names(self) -> t.List[str]: + def get_template_names(self) -> list[str]: """Checks if the view uses the page template or listing template.""" if self.page_path.is_file(): template_name = "content/page.html" @@ -36,7 +35,7 @@ class PageOrCategoryView(TemplateView): return [template_name] - def get_context_data(self, **kwargs) -> t.Dict[str, t.Any]: + def get_context_data(self, **kwargs) -> dict[str, any]: """Assign proper context variables based on what resource user requests.""" context = super().get_context_data(**kwargs) @@ -73,7 +72,7 @@ class PageOrCategoryView(TemplateView): return context @staticmethod - def _get_page_context(path: Path) -> t.Dict[str, t.Any]: + def _get_page_context(path: Path) -> dict[str, any]: page, metadata = utils.get_page(path) return { "page": page, @@ -84,7 +83,7 @@ class PageOrCategoryView(TemplateView): } @staticmethod - def _get_category_context(path: Path) -> t.Dict[str, t.Any]: + def _get_category_context(path: Path) -> dict[str, any]: category = utils.get_category(path) return { "categories": utils.get_categories(path), diff --git a/pydis_site/templates/content/listing.html b/pydis_site/templates/content/listing.html index ef0ef919..eeb6b5e2 100644 --- a/pydis_site/templates/content/listing.html +++ b/pydis_site/templates/content/listing.html @@ -1,6 +1,8 @@ +{# Base navigation screen for resources #} {% extends 'content/base.html' %} {% block page_content %} + {# Nested Categories #} {% for category, data in categories.items %}
@@ -13,6 +15,8 @@

{{ data.description }}

{% endfor %} + + {# Single Pages #} {% for page, data in pages.items %}
@@ -21,7 +25,11 @@ {{ data.title }} -

{{ data.description }}

+ {% if "tags" in location %} +

{{ data.description | safe }}

+ {% else %} +

{{ data.description }}

+ {% endif %}
{% endfor %} {% endblock %} diff --git a/pydis_site/templates/content/page.html b/pydis_site/templates/content/page.html index 759286f6..625c01f1 100644 --- a/pydis_site/templates/content/page.html +++ b/pydis_site/templates/content/page.html @@ -5,7 +5,7 @@ - + {% endblock %} {% block page_content %} -- cgit v1.2.3 From 42124deb7ea5f17bc6faf959baba8e951b567655 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 06:10:38 +0200 Subject: Add Tag Page Template Add a template for the tag page itself, and add a route to use it. Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/urls.py | 21 ++++++++++-- pydis_site/apps/content/views/__init__.py | 3 +- pydis_site/apps/content/views/tags.py | 56 +++++++++++++++++++++++++++++++ pydis_site/static/css/content/tag.css | 7 ++++ pydis_site/templates/content/base.html | 2 +- pydis_site/templates/content/tag.html | 21 ++++++++++++ 6 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 pydis_site/apps/content/views/tags.py create mode 100644 pydis_site/static/css/content/tag.css create mode 100644 pydis_site/templates/content/tag.html diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index f8496095..b4ffc07d 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -3,7 +3,7 @@ from pathlib import Path from django_distill import distill_path -from . import views +from . import utils, views app_name = "content" @@ -29,18 +29,33 @@ def __get_all_files(root: Path, folder: typing.Optional[Path] = None) -> list[st return results -def get_all_pages() -> typing.Iterator[dict[str, str]]: +DISTILL_RETURN = typing.Iterator[dict[str, str]] + + +def get_all_pages() -> DISTILL_RETURN: """Yield a dict of all page categories.""" for location in __get_all_files(Path("pydis_site", "apps", "content", "resources")): yield {"location": location} +def get_all_tags() -> DISTILL_RETURN: + """Return all tag names in the repository in static builds.""" + for tag in utils.get_tags_static(): + yield {"name": tag.name} + + urlpatterns = [ distill_path("", views.PageOrCategoryView.as_view(), name='pages'), + distill_path( + "tags//", + views.TagView.as_view(), + name="tag", + distill_func=get_all_tags + ), distill_path( "/", views.PageOrCategoryView.as_view(), name='page_category', distill_func=get_all_pages - ), + ) ] diff --git a/pydis_site/apps/content/views/__init__.py b/pydis_site/apps/content/views/__init__.py index 70ea1c7a..a969b1dc 100644 --- a/pydis_site/apps/content/views/__init__.py +++ b/pydis_site/apps/content/views/__init__.py @@ -1,3 +1,4 @@ from .page_category import PageOrCategoryView +from .tags import TagView -__all__ = ["PageOrCategoryView"] +__all__ = ["PageOrCategoryView", "TagView"] diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py new file mode 100644 index 00000000..12e311dc --- /dev/null +++ b/pydis_site/apps/content/views/tags.py @@ -0,0 +1,56 @@ +import re + +import frontmatter +import markdown +from django.conf import settings +from django.http import Http404 +from django.urls import reverse +from django.views.generic import TemplateView + +from pydis_site.apps.content import utils +from pydis_site.apps.content.models.tag import Tag + +COMMAND_REGEX = re.compile(r"`*!tags? (?P[\w\d-]+)`*") + + +class TagView(TemplateView): + """Handles tag pages.""" + + template_name = "content/tag.html" + + def get_context_data(self, **kwargs) -> dict: + """Get the relevant context for this tag page.""" + try: + tag = utils.get_tag(kwargs.get("name")) + except Tag.DoesNotExist: + raise Http404 + + context = super().get_context_data(**kwargs) + context["page_title"] = tag.name + body = frontmatter.parse(tag.body) + content = body[1] + + # Check for tags which can be hyperlinked + start = 0 + while match := COMMAND_REGEX.search(content, start): + link = reverse("content:tag", kwargs={"name": match.group("name")}) + content = content[:match.start()] + f"[{match.group()}]({link})" + content[match.end():] + start = match.end() + + # Add support for some embed elements + if embed := body[0].get("embed"): + context["page_title"] = embed["title"] + if image := embed.get("image"): + content = f"![{embed['title']}]({image['url']})\n\n" + content + + context.update({ + "page": markdown.markdown(content, extensions=["pymdownx.superfences"]), + "tag": tag, + }) + + context["breadcrumb_items"] = [{ + "name": utils.get_category(settings.CONTENT_PAGES_PATH / location)["title"], + "path": str(location) + } for location in [".", "tags"]] + + return context diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css new file mode 100644 index 00000000..a144ce24 --- /dev/null +++ b/pydis_site/static/css/content/tag.css @@ -0,0 +1,7 @@ +h1.title a { + color: black; +} + +h1.title a:hover { + color: #7289DA; +} diff --git a/pydis_site/templates/content/base.html b/pydis_site/templates/content/base.html index 4a19a275..dbd303a1 100644 --- a/pydis_site/templates/content/base.html +++ b/pydis_site/templates/content/base.html @@ -35,7 +35,7 @@
-

{{ page_title }}

+

{% block title_element %}{{ page_title }}{% endblock %}

{% block page_content %}{% endblock %}
diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html new file mode 100644 index 00000000..264f63d0 --- /dev/null +++ b/pydis_site/templates/content/tag.html @@ -0,0 +1,21 @@ +{% extends "content/page.html" %} +{% load static %} + +{% block head %} + {{ block.super }} + + {{ tag.name }} +{% endblock %} + +{% block title_element %} +
+
{{ block.super }}
+
+ +
+
+{% endblock %} + +{% block page_content %} + {{ block.super }} +{% endblock %} -- cgit v1.2.3 From 79bc1760581f42151923bacff0a3d97037f2893a Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 06:21:05 +0200 Subject: Add PyMDown Extensions Adds the python-markdown extension pack as it offers better support for things such as codeblocks. Signed-off-by: Hassan Abouelela --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 1bee4397..d335f271 100644 --- a/poetry.lock +++ b/poetry.lock @@ -692,6 +692,17 @@ dev = ["sphinx", "sphinx-rtd-theme", "zope.interface", "cryptography (>=3.3.1)", docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"] +[[package]] +name = "pymdown-extensions" +version = "9.5" +description = "Extension pack for Python Markdown." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +markdown = ">=3.2" + [[package]] name = "python-dotenv" version = "0.17.1" @@ -931,7 +942,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "c656c07f40d32ee7d30c19a7084b40e1e851209a362a3fe882aa03c2fd286454" +content-hash = "61811384e91a5b9fc2930486a0862b0bcb476fcd9ad6cd72287f768e945310e8" [metadata.files] anyio = [ @@ -1231,6 +1242,10 @@ pyjwt = [ {file = "PyJWT-2.4.0-py3-none-any.whl", hash = "sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf"}, {file = "PyJWT-2.4.0.tar.gz", hash = "sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba"}, ] +pymdown-extensions = [ + {file = "pymdown_extensions-9.5-py3-none-any.whl", hash = "sha256:ec141c0f4983755349f0c8710416348d1a13753976c028186ed14f190c8061c4"}, + {file = "pymdown_extensions-9.5.tar.gz", hash = "sha256:3ef2d998c0d5fa7eb09291926d90d69391283561cf6306f85cd588a5eb5befa0"}, +] python-dotenv = [ {file = "python-dotenv-0.17.1.tar.gz", hash = "sha256:b1ae5e9643d5ed987fc57cc2583021e38db531946518130777734f9589b3141f"}, {file = "python_dotenv-0.17.1-py2.py3-none-any.whl", hash = "sha256:00aa34e92d992e9f8383730816359647f358f4a3be1ba45e5a5cefd27ee91544"}, diff --git a/pyproject.toml b/pyproject.toml index 037f837c..831f7ad5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ python-frontmatter = "~=1.0" django-prometheus = "~=2.1" django-distill = "~=2.9.0" PyJWT = {version = "~=2.4.0", extras = ["crypto"]} +pymdown-extensions = "9.5" [tool.poetry.dev-dependencies] coverage = "~=5.0" -- cgit v1.2.3 From 5dbb1d29242435d4fa13dc09ac285b3522893eac Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 06:25:52 +0200 Subject: Add Content App To Navbar Signed-off-by: Hassan Abouelela --- pydis_site/templates/base/navbar.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydis_site/templates/base/navbar.html b/pydis_site/templates/base/navbar.html index d7fb4f4c..931693c8 100644 --- a/pydis_site/templates/base/navbar.html +++ b/pydis_site/templates/base/navbar.html @@ -67,6 +67,9 @@ Resources + + Content + Events -- cgit v1.2.3 From 733d5c084b1cba91a76f495b37ef0a391e5f9900 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 06:43:05 +0200 Subject: Export Tag Model As Top Level Model Object Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/models/__init__.py | 3 +++ pydis_site/apps/content/utils.py | 2 +- pydis_site/apps/content/views/tags.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py index e69de29b..2718ce94 100644 --- a/pydis_site/apps/content/models/__init__.py +++ b/pydis_site/apps/content/models/__init__.py @@ -0,0 +1,3 @@ +from .tag import Tag + +__all__ = ["Tag"] diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index a4252284..de609596 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -14,7 +14,7 @@ from django.utils import timezone from markdown.extensions.toc import TocExtension from pydis_site import settings -from .models.tag import Tag +from .models import Tag TAG_URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" TAG_CACHE_TTL = datetime.timedelta(hours=1) diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py index 12e311dc..e2cfb488 100644 --- a/pydis_site/apps/content/views/tags.py +++ b/pydis_site/apps/content/views/tags.py @@ -8,7 +8,7 @@ from django.urls import reverse from django.views.generic import TemplateView from pydis_site.apps.content import utils -from pydis_site.apps.content.models.tag import Tag +from pydis_site.apps.content.models import Tag COMMAND_REGEX = re.compile(r"`*!tags? (?P[\w\d-]+)`*") -- cgit v1.2.3 From a8fd8b823748ced96bb8ea34e8dcd8bd0dd57671 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 08:47:29 +0200 Subject: Add Tag View Tests Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/tests/test_utils.py | 103 +++++++++++++++++++++- pydis_site/apps/content/tests/test_views.py | 128 ++++++++++++++++++++++++++++ pydis_site/apps/content/utils.py | 6 +- 3 files changed, 233 insertions(+), 4 deletions(-) diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index be5ea897..89ef81c4 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -1,8 +1,16 @@ +import tarfile +import tempfile +import textwrap from pathlib import Path +from unittest import mock +import httpx +import markdown from django.http import Http404 +from django.test import TestCase -from pydis_site.apps.content import utils +from pydis_site import settings +from pydis_site.apps.content import models, utils from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) @@ -96,3 +104,96 @@ class GetPageTests(MockPagesTestCase): def test_get_nonexistent_page_returns_404(self): with self.assertRaises(Http404): utils.get_page(Path(BASE_PATH, "invalid")) + + +class TagUtilsTests(TestCase): + """Tests for the tag-related utilities.""" + + @mock.patch.object(utils, "fetch_tags") + def test_static_fetch(self, fetch_mock: mock.Mock): + """Test that the static fetch function is only called at most once during static builds.""" + tags = [models.Tag(name="Name", body="body", url="url")] + fetch_mock.return_value = tags + result = utils.get_tags_static() + second_result = utils.get_tags_static() + + fetch_mock.assert_called_once() + self.assertEqual(tags, result) + self.assertEqual(tags, second_result) + + @mock.patch("httpx.get") + def test_mocked_fetch(self, get_mock: mock.Mock): + """Test that proper data is returned from fetch, but with a mocked API response.""" + bodies = ( + "This is the first tag!", + textwrap.dedent(""" + --- + frontmatter: empty + --- + This tag has frontmatter! + """), + ) + + # Generate a tar archive with a few tags + with tempfile.TemporaryDirectory() as tar_folder: + tar_folder = Path(tar_folder) + with tempfile.TemporaryDirectory() as folder: + folder = Path(folder) + (folder / "ignored_file.md").write_text("This is an ignored file.") + tags_folder = folder / "bot/resources/tags" + tags_folder.mkdir(parents=True) + + (tags_folder / "first_tag.md").write_text(bodies[0]) + (tags_folder / "second_tag.md").write_text(bodies[1]) + + with tarfile.open(tar_folder / "temp.tar", "w") as file: + file.add(folder, recursive=True) + + body = (tar_folder / "temp.tar").read_bytes() + + get_mock.return_value = httpx.Response( + status_code=200, + content=body, + request=httpx.Request("GET", "https://google.com"), + ) + + result = utils.fetch_tags() + self.assertEqual([ + models.Tag(name="first_tag", body=bodies[0], url=f"{utils.TAG_URL_BASE}/first_tag.md"), + models.Tag(name="second_tag", body=bodies[1], url=f"{utils.TAG_URL_BASE}/first_tag.md"), + ], sorted(result, key=lambda tag: tag.name)) + + def test_get_real_tag(self): + """Test that a single tag is returned if it exists.""" + tag = models.Tag.objects.create(name="real-tag") + result = utils.get_tag("real-tag") + + self.assertEqual(tag, result) + + def test_get_tag_404(self): + """Test that an error is raised when we fetch a non-existing tag.""" + models.Tag.objects.create(name="real-tag") + with self.assertRaises(models.Tag.DoesNotExist): + utils.get_tag("fake") + + def test_category_pages(self): + """Test that the category pages function returns the correct records for tags.""" + models.Tag.objects.create(name="second-tag", body="Normal body") + models.Tag.objects.create(name="first-tag", body="Normal body") + tag_body = {"description": markdown.markdown("Normal body"), "icon": "fas fa-tag"} + + result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") + self.assertDictEqual({ + "first-tag": {**tag_body, "title": "first-tag"}, + "second-tag": {**tag_body, "title": "second-tag"}, + }, result) + + def test_trimmed_tag_content(self): + """Test a tag with a long body that requires trimming.""" + tag = models.Tag.objects.create(name="long-tag", body="E" * 300) + result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") + self.assertDictEqual({"long-tag": { + "title": "long-tag", + "description": markdown.markdown(tag.body[:100] + "..."), + "icon": "fas fa-tag", + }}, result) diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index eadad7e3..a5867260 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -1,9 +1,14 @@ +import textwrap from pathlib import Path from unittest import TestCase +import django.test +import markdown from django.http import Http404 from django.test import RequestFactory, SimpleTestCase, override_settings +from django.urls import reverse +from pydis_site.apps.content.models import Tag from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) @@ -180,3 +185,126 @@ class PageOrCategoryViewTests(MockPagesTestCase, SimpleTestCase, TestCase): {"name": PARSED_CATEGORY_INFO["title"], "path": Path("category/subcategory")}, ] ) + + +class TagViewTests(django.test.TestCase): + """Tests for the TagView class.""" + + def setUp(self): + """Set test helpers, then set up fake filesystem.""" + super().setUp() + + def test_valid_tag_returns_200(self): + """Test that a page is returned for a valid tag.""" + Tag.objects.create(name="example", body="This is the tag body.", url="URL") + response = self.client.get("/pages/tags/example/") + self.assertEqual(200, response.status_code) + self.assertIn("This is the tag body", response.content.decode("utf-8")) + self.assertTemplateUsed(response, "content/tag.html") + + def test_invalid_tag_404(self): + """Test that a tag which doesn't exist raises a 404.""" + response = self.client.get("/pages/tags/non-existent/") + self.assertEqual(404, response.status_code) + + def test_context(self): + """Check that the context contains all the necessary data.""" + body = textwrap.dedent(""" + --- + unused: frontmatter + ---- + Tag content here. + """) + + tag = Tag.objects.create(name="example", body=body, url="URL") + response = self.client.get("/pages/tags/example/") + expected = { + "page_title": "example", + "page": markdown.markdown("Tag content here."), + "tag": tag, + } + for key in expected: + self.assertEqual( + expected[key], response.context.get(key), f"context.{key} did not match" + ) + + def test_markdown(self): + """Test that markdown content is rendered properly.""" + body = textwrap.dedent(""" + ```py + Hello world! + ``` + + **This text is in bold** + """) + + Tag.objects.create(name="example", body=body, url="URL") + response = self.client.get("/pages/tags/example/") + content = response.content.decode("utf-8") + + self.assertInHTML('Hello world!', content) + self.assertInHTML("This text is in bold", content) + + def test_embed(self): + """Test that an embed from the frontmatter is treated correctly.""" + body = textwrap.dedent(""" + --- + embed: + title: Embed title + image: + url: https://google.com + --- + Tag body. + """) + + Tag.objects.create(name="example", body=body, url="URL") + response = self.client.get("/pages/tags/example/") + content = response.content.decode("utf-8") + + self.assertInHTML('Embed title', content) + self.assertInHTML("

Tag body.

", content) + + def test_embed_title(self): + """Test that the page title gets set to the embed title.""" + body = textwrap.dedent(""" + --- + embed: + title: Embed title + --- + """) + + Tag.objects.create(name="example", body=body, url="URL") + response = self.client.get("/pages/tags/example/") + self.assertEqual( + "Embed title", + response.context.get("page_title"), + "The page title must match the embed title." + ) + + def test_hyperlinked_item(self): + """Test hyperlinking of tags works as intended.""" + filler_before, filler_after = "empty filler text\n\n", "more\nfiller" + body = filler_before + "`!tags return`" + filler_after + Tag.objects.create(name="example", body=body, url="URL") + + other_url = reverse("content:tag", kwargs={"name": "return"}) + response = self.client.get("/pages/tags/example/") + self.assertEqual( + markdown.markdown(filler_before + f"[`!tags return`]({other_url})" + filler_after), + response.context.get("page") + ) + + def test_tag_root_page(self): + """Test the root tag page which lists all tags.""" + Tag.objects.create(name="tag-1") + Tag.objects.create(name="tag-2") + Tag.objects.create(name="tag-3") + + response = self.client.get("/pages/tags/") + content = response.content.decode("utf-8") + + self.assertTemplateUsed(response, "content/listing.html") + self.assertInHTML('

Tags

', content) + + for tag_number in range(1, 4): + self.assertIn(f"tag-{tag_number}", content) diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index de609596..76437593 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -56,7 +56,7 @@ def fetch_tags() -> list[Tag]: The entire repository is downloaded and extracted locally because getting file content would require one request per file, and can get rate-limited. """ - if settings.GITHUB_TOKEN: + if settings.GITHUB_TOKEN: # pragma: no cover headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} else: headers = {} @@ -90,7 +90,7 @@ def fetch_tags() -> list[Tag]: def get_tags() -> list[Tag]: """Return a list of all tags visible to the application, from the cache or API.""" - if settings.STATIC_BUILD: + if settings.STATIC_BUILD: # pragma: no cover last_update = None else: last_update = ( @@ -100,7 +100,7 @@ def get_tags() -> list[Tag]: if last_update is None or timezone.now() >= (last_update + TAG_CACHE_TTL): # Stale or empty cache - if settings.STATIC_BUILD: + if settings.STATIC_BUILD: # pragma: no cover tags = get_tags_static() else: tags = fetch_tags() -- cgit v1.2.3 From db24ace579d70d50970f9503997148e063d63392 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 21:52:02 +0200 Subject: Set Link Color For Sub-elements Signed-off-by: Hassan Abouelela --- pydis_site/static/css/content/tag.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css index a144ce24..a3db046c 100644 --- a/pydis_site/static/css/content/tag.css +++ b/pydis_site/static/css/content/tag.css @@ -5,3 +5,9 @@ h1.title a { h1.title a:hover { color: #7289DA; } + +.content a * { + /* This is the original color, but propagated down the chain */ + /* which allows for elements inside links, such as codeblocks */ + color: #7289DA; +} -- cgit v1.2.3 From 74226511e22f04ae5a0f0975849469ad027dc963 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 21:54:46 +0200 Subject: Clean Up Tag Link Substitution Co-authored-by: ChrisJL Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/views/tags.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py index e2cfb488..5295537d 100644 --- a/pydis_site/apps/content/views/tags.py +++ b/pydis_site/apps/content/views/tags.py @@ -31,11 +31,10 @@ class TagView(TemplateView): content = body[1] # Check for tags which can be hyperlinked - start = 0 - while match := COMMAND_REGEX.search(content, start): + def sub(match: re.Match) -> str: link = reverse("content:tag", kwargs={"name": match.group("name")}) - content = content[:match.start()] + f"[{match.group()}]({link})" + content[match.end():] - start = match.end() + return f"[{match.group()}]({link})" + content = COMMAND_REGEX.sub(sub, content) # Add support for some embed elements if embed := body[0].get("embed"): -- cgit v1.2.3 From 95da22074e6f2c7b2425ccca75cd25f9ef5596fb Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 22:49:36 +0200 Subject: Enable Code Highlighting On All Content Pages Signed-off-by: Hassan Abouelela --- pydis_site/templates/content/base.html | 4 ++++ pydis_site/templates/content/page.html | 8 -------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pydis_site/templates/content/base.html b/pydis_site/templates/content/base.html index dbd303a1..2fd721a3 100644 --- a/pydis_site/templates/content/base.html +++ b/pydis_site/templates/content/base.html @@ -8,6 +8,10 @@ + + + {% endblock %} diff --git a/pydis_site/templates/content/page.html b/pydis_site/templates/content/page.html index 625c01f1..679ecec6 100644 --- a/pydis_site/templates/content/page.html +++ b/pydis_site/templates/content/page.html @@ -1,13 +1,5 @@ {% extends 'content/base.html' %} -{% block head %} - {{ block.super }} - - - -{% endblock %} - {% block page_content %} {% if relevant_links or toc %}
-- cgit v1.2.3 From f2ad3eed8ef8872713666f69ec783f59006d3d81 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 13 Aug 2022 22:53:50 +0200 Subject: Improve Tag Cropping Move the tag cropping logic to the frontend, which makes it easier to crop without crossing boundaries such as link or code block boundaries. Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/utils.py | 6 +----- pydis_site/static/js/content/listing.js | 36 +++++++++++++++++++++++++++++++ pydis_site/templates/content/listing.html | 4 +++- 3 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 pydis_site/static/js/content/listing.js diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index 76437593..cc08f81f 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -131,14 +131,10 @@ def get_category_pages(path: Path) -> dict[str, dict]: tags = {} for tag in get_tags(): content = frontmatter.parse(tag.body)[1] - if len(content) > 100: - # Trim the preview to a maximum of 100 visible characters - # This causes some markdown to break, but we ignore that - content = content[:100] + "..." tags[tag.name] = { "title": tag.name, - "description": markdown.markdown(content), + "description": markdown.markdown(content, extensions=["pymdownx.superfences"]), "icon": "fas fa-tag" } diff --git a/pydis_site/static/js/content/listing.js b/pydis_site/static/js/content/listing.js new file mode 100644 index 00000000..3502cb2a --- /dev/null +++ b/pydis_site/static/js/content/listing.js @@ -0,0 +1,36 @@ +/** + * Trim a tag listing to only show a few lines of content. + */ +function trimTag() { + const containers = document.getElementsByClassName("tag-container"); + for (const container of containers) { + // Remove every element after the first two paragraphs + while (container.children.length > 2) { + container.removeChild(container.lastChild); + } + + // Trim down the elements if they are too long + const containerLength = container.textContent.length; + if (containerLength > 300) { + if (containerLength - container.firstChild.textContent.length > 300) { + // The first element alone takes up more than 300 characters + container.removeChild(container.lastChild); + } + + let last = container.lastChild.lastChild; + while (container.textContent.length > 300 && container.lastChild.childNodes.length > 0) { + last = container.lastChild.lastChild; + last.remove(); + } + + if (container.textContent.length > 300 && (last instanceof HTMLElement && last.tagName !== "CODE")) { + // Add back the final element (up to a period if possible) + const stop = last.textContent.indexOf("."); + last.textContent = last.textContent.slice(0, stop > 0 ? stop + 1: null); + container.lastChild.appendChild(last); + } + } + } +} + +trimTag(); diff --git a/pydis_site/templates/content/listing.html b/pydis_site/templates/content/listing.html index eeb6b5e2..098f4237 100644 --- a/pydis_site/templates/content/listing.html +++ b/pydis_site/templates/content/listing.html @@ -1,5 +1,6 @@ {# Base navigation screen for resources #} {% extends 'content/base.html' %} +{% load static %} {% block page_content %} {# Nested Categories #} @@ -26,10 +27,11 @@ {{ data.title }} {% if "tags" in location %} -

{{ data.description | safe }}

+
{{ data.description | safe }}
{% else %}

{{ data.description }}

{% endif %}
{% endfor %} + {% endblock %} -- cgit v1.2.3 From b4911d03faf8eecf5c4cced6f8036b0b2ef01d58 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 14 Aug 2022 04:44:57 +0200 Subject: Move Tag URL To Property And Add Group The URLs can be simply constructed using the other tag properties, so they were removed from the database in favor of a property. A group field was also added to support tags within groups. Signed-off-by: Hassan Abouelela --- .../migrations/0002_remove_tag_url_tag_group.py | 22 ++++++++++++++++++++++ pydis_site/apps/content/models/tag.py | 17 ++++++++++++++++- pydis_site/apps/content/tests/test_utils.py | 6 +++--- pydis_site/apps/content/tests/test_views.py | 12 ++++++------ 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py diff --git a/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py b/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py new file mode 100644 index 00000000..e59077f0 --- /dev/null +++ b/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.6 on 2022-08-13 23:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('content', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='tag', + name='url', + ), + migrations.AddField( + model_name='tag', + name='group', + field=models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True), + ), + ] diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py index 1437b96a..01264ff1 100644 --- a/pydis_site/apps/content/models/tag.py +++ b/pydis_site/apps/content/models/tag.py @@ -4,6 +4,8 @@ from django.db import models class Tag(models.Model): """A tag from the python-discord server.""" + URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" + last_updated = models.DateTimeField( help_text="The date and time this data was last fetched.", auto_now=True, @@ -13,5 +15,18 @@ class Tag(models.Model): primary_key=True, max_length=50, ) + group = models.CharField( + help_text="The group the tag belongs to.", + null=True, + max_length=50, + ) body = models.TextField(help_text="The content of the tag.") - url = models.URLField(help_text="The URL to this tag on GitHub.") + + @property + def url(self) -> str: + """Get the URL of the tag on GitHub.""" + url = Tag.URL_BASE + if self.group: + url += f"/{self.group}" + url += f"/{self.name}.md" + return url diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index 89ef81c4..a5d5dcb4 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -112,7 +112,7 @@ class TagUtilsTests(TestCase): @mock.patch.object(utils, "fetch_tags") def test_static_fetch(self, fetch_mock: mock.Mock): """Test that the static fetch function is only called at most once during static builds.""" - tags = [models.Tag(name="Name", body="body", url="url")] + tags = [models.Tag(name="Name", body="body")] fetch_mock.return_value = tags result = utils.get_tags_static() second_result = utils.get_tags_static() @@ -159,8 +159,8 @@ class TagUtilsTests(TestCase): result = utils.fetch_tags() self.assertEqual([ - models.Tag(name="first_tag", body=bodies[0], url=f"{utils.TAG_URL_BASE}/first_tag.md"), - models.Tag(name="second_tag", body=bodies[1], url=f"{utils.TAG_URL_BASE}/first_tag.md"), + models.Tag(name="first_tag", body=bodies[0]), + models.Tag(name="second_tag", body=bodies[1]), ], sorted(result, key=lambda tag: tag.name)) def test_get_real_tag(self): diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index a5867260..c4d3474e 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -196,7 +196,7 @@ class TagViewTests(django.test.TestCase): def test_valid_tag_returns_200(self): """Test that a page is returned for a valid tag.""" - Tag.objects.create(name="example", body="This is the tag body.", url="URL") + Tag.objects.create(name="example", body="This is the tag body.") response = self.client.get("/pages/tags/example/") self.assertEqual(200, response.status_code) self.assertIn("This is the tag body", response.content.decode("utf-8")) @@ -216,7 +216,7 @@ class TagViewTests(django.test.TestCase): Tag content here. """) - tag = Tag.objects.create(name="example", body=body, url="URL") + tag = Tag.objects.create(name="example", body=body) response = self.client.get("/pages/tags/example/") expected = { "page_title": "example", @@ -238,7 +238,7 @@ class TagViewTests(django.test.TestCase): **This text is in bold** """) - Tag.objects.create(name="example", body=body, url="URL") + Tag.objects.create(name="example", body=body) response = self.client.get("/pages/tags/example/") content = response.content.decode("utf-8") @@ -257,7 +257,7 @@ class TagViewTests(django.test.TestCase): Tag body. """) - Tag.objects.create(name="example", body=body, url="URL") + Tag.objects.create(name="example", body=body) response = self.client.get("/pages/tags/example/") content = response.content.decode("utf-8") @@ -273,7 +273,7 @@ class TagViewTests(django.test.TestCase): --- """) - Tag.objects.create(name="example", body=body, url="URL") + Tag.objects.create(name="example", body=body) response = self.client.get("/pages/tags/example/") self.assertEqual( "Embed title", @@ -285,7 +285,7 @@ class TagViewTests(django.test.TestCase): """Test hyperlinking of tags works as intended.""" filler_before, filler_after = "empty filler text\n\n", "more\nfiller" body = filler_before + "`!tags return`" + filler_after - Tag.objects.create(name="example", body=body, url="URL") + Tag.objects.create(name="example", body=body) other_url = reverse("content:tag", kwargs={"name": "return"}) response = self.client.get("/pages/tags/example/") -- cgit v1.2.3 From 45cdb27a82297ede18d7bd908213dde54fef06a9 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 14 Aug 2022 05:34:27 +0200 Subject: Add Tag Group Support Adds support for tag groups in content. This involves some modification to the routing, and templating. Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/urls.py | 12 ++- pydis_site/apps/content/utils.py | 101 ++++++++++++++++++++----- pydis_site/apps/content/views/page_category.py | 5 +- pydis_site/apps/content/views/tags.py | 79 +++++++++++++++---- pydis_site/static/css/content/color.css | 7 ++ pydis_site/static/css/content/tag.css | 8 -- pydis_site/static/js/content/listing.js | 5 ++ pydis_site/templates/content/listing.html | 17 ++++- pydis_site/templates/content/tag.html | 5 +- 9 files changed, 187 insertions(+), 52 deletions(-) create mode 100644 pydis_site/static/css/content/color.css diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index b4ffc07d..03c0015a 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -39,15 +39,21 @@ def get_all_pages() -> DISTILL_RETURN: def get_all_tags() -> DISTILL_RETURN: - """Return all tag names in the repository in static builds.""" + """Return all tag names and groups in static builds.""" + groups = {None} for tag in utils.get_tags_static(): - yield {"name": tag.name} + groups.add(tag.group) + yield {"location": (f"{tag.group}/" if tag.group else "") + tag.name} + + groups.remove(None) + for group in groups: + yield {"location": group} urlpatterns = [ distill_path("", views.PageOrCategoryView.as_view(), name='pages'), distill_path( - "tags//", + "tags//", views.TagView.as_view(), name="tag", distill_func=get_all_tags diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index cc08f81f..da6a024d 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -2,6 +2,7 @@ import datetime import functools import tarfile import tempfile +import typing from io import BytesIO from pathlib import Path @@ -16,7 +17,6 @@ from markdown.extensions.toc import TocExtension from pydis_site import settings from .models import Tag -TAG_URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" TAG_CACHE_TTL = datetime.timedelta(hours=1) @@ -44,9 +44,13 @@ def get_tags_static() -> list[Tag]: """ Fetch tag information in static builds. + This also includes some fake tags to preview the tag groups feature. This will return a cached value, so it should only be used for static builds. """ - return fetch_tags() + tags = fetch_tags() + for tag in tags[3:5]: + tag.group = "very-cool-group" + return tags def fetch_tags() -> list[Tag]: @@ -79,10 +83,15 @@ def fetch_tags() -> list[Tag]: repo.extractall(folder, included) for tag_file in Path(folder).rglob("*.md"): + group = None + if tag_file.parent.name != "tags": + # Tags in sub-folders are considered part of a group + group = tag_file.parent.name + tags.append(Tag( name=tag_file.name.removesuffix(".md"), + group=group, body=tag_file.read_text(encoding="utf-8"), - url=f"{TAG_URL_BASE}/{tag_file.name}" )) return tags @@ -114,31 +123,85 @@ def get_tags() -> list[Tag]: return Tag.objects.all() -def get_tag(name: str) -> Tag: - """Return a tag by name.""" - tags = get_tags() - for tag in tags: - if tag.name == name: +def get_tag(path: str) -> typing.Union[Tag, list[Tag]]: + """ + Return a tag based on the search location. + + The tag name and group must match. If only one argument is provided in the path, + it's assumed to either be a group name, or a no-group tag name. + + If it's a group name, a list of tags which belong to it is returned. + """ + path = path.split("/") + if len(path) == 2: + group, name = path[0], path[1] + else: + name = path[0] + group = None + + matches = [] + for tag in get_tags(): + if tag.name == name and tag.group == group: return tag + elif tag.group == name and group is None: + matches.append(tag) + + if matches: + return matches raise Tag.DoesNotExist() -def get_category_pages(path: Path) -> dict[str, dict]: - """Get all page names and their metadata at a category path.""" - # Special handling for tags - if path == Path(__file__).parent / "resources/tags": - tags = {} - for tag in get_tags(): - content = frontmatter.parse(tag.body)[1] +def get_tag_category( + tags: typing.Optional[list[Tag]] = None, *, collapse_groups: bool +) -> dict[str, dict]: + """ + Generate context data for `tags`, or all tags if None. + + If `tags` is None, `get_tag` is used to populate the data. + If `collapse_groups` is True, tags with parent groups are not included in the list, + and instead the parent itself is included as a single entry with it's sub-tags + in the description. + """ + if not tags: + tags = get_tags() + + data = [] + groups = {} - tags[tag.name] = { + # Create all the metadata for the tags + for tag in tags: + if tag.group is None or not collapse_groups: + content = frontmatter.parse(tag.body)[1] + data.append({ "title": tag.name, "description": markdown.markdown(content, extensions=["pymdownx.superfences"]), - "icon": "fas fa-tag" - } + "icon": "fas fa-tag", + }) + else: + if tag.group not in groups: + groups[tag.group] = { + "title": tag.group, + "description": [tag.name], + "icon": "fas fa-tags", + } + else: + groups[tag.group]["description"].append(tag.name) - return {name: tags[name] for name in sorted(tags)} + # Flatten group description into a single string + for group in groups.values(): + group["description"] = "Contains the following tags: " + ", ".join(group["description"]) + data.append(group) + + # Sort the tags, and return them in the proper format + return {tag["title"]: tag for tag in sorted(data, key=lambda tag: tag["title"].lower())} + + +def get_category_pages(path: Path) -> dict[str, dict]: + """Get all page names and their metadata at a category path.""" + # Special handling for tags + if path == Path(__file__).parent / "resources/tags": + return get_tag_category(collapse_groups=True) pages = {} diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py index 01ce8402..062c2bc1 100644 --- a/pydis_site/apps/content/views/page_category.py +++ b/pydis_site/apps/content/views/page_category.py @@ -5,7 +5,7 @@ from django.conf import settings from django.http import Http404, HttpRequest, HttpResponse from django.views.generic import TemplateView -from pydis_site.apps.content import utils +from pydis_site.apps.content import models, utils class PageOrCategoryView(TemplateView): @@ -91,4 +91,7 @@ class PageOrCategoryView(TemplateView): "page_title": category["title"], "page_description": category["description"], "icon": category.get("icon"), + "app_name": "content:page_category", + "is_tag_listing": "/resources/tags" in path.as_posix(), + "tag_url": models.Tag.URL_BASE, } diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py index 5295537d..a8df65db 100644 --- a/pydis_site/apps/content/views/tags.py +++ b/pydis_site/apps/content/views/tags.py @@ -1,4 +1,5 @@ import re +import typing import frontmatter import markdown @@ -16,23 +17,65 @@ COMMAND_REGEX = re.compile(r"`*!tags? (?P[\w\d-]+)`*") class TagView(TemplateView): """Handles tag pages.""" - template_name = "content/tag.html" + tag: typing.Union[Tag, list[Tag]] + is_group: bool + + def setup(self, *args, **kwargs) -> None: + """Look for a tag, and configure the view.""" + super().setup(*args, **kwargs) - def get_context_data(self, **kwargs) -> dict: - """Get the relevant context for this tag page.""" try: - tag = utils.get_tag(kwargs.get("name")) + self.tag = utils.get_tag(kwargs.get("location")) + self.is_group = isinstance(self.tag, list) except Tag.DoesNotExist: raise Http404 + def get_template_names(self) -> list[str]: + """Either return the tag page template, or the listing.""" + if self.is_group: + template_name = "content/listing.html" + else: + template_name = "content/tag.html" + + return [template_name] + + def get_context_data(self, **kwargs) -> dict: + """Get the relevant context for this tag page or group.""" context = super().get_context_data(**kwargs) - context["page_title"] = tag.name + context["breadcrumb_items"] = [{ + "name": utils.get_category(settings.CONTENT_PAGES_PATH / location)["title"], + "path": location, + } for location in (".", "tags")] + + if self.is_group: + self._set_group_context(context, self.tag) + else: + self._set_tag_context(context, self.tag) + + return context + + @staticmethod + def _set_tag_context(context: dict[str, any], tag: Tag) -> None: + """Update the context with the information for a tag page.""" + context.update({ + "page_title": tag.name, + "tag": tag, + }) + + if tag.group: + # Add group names to the breadcrumbs + context["breadcrumb_items"].append({ + "name": tag.group, + "path": f"tags/{tag.group}", + }) + + # Clean up tag body body = frontmatter.parse(tag.body) content = body[1] # Check for tags which can be hyperlinked def sub(match: re.Match) -> str: - link = reverse("content:tag", kwargs={"name": match.group("name")}) + link = reverse("content:tag", kwargs={"location": match.group("name")}) return f"[{match.group()}]({link})" content = COMMAND_REGEX.sub(sub, content) @@ -42,14 +85,20 @@ class TagView(TemplateView): if image := embed.get("image"): content = f"![{embed['title']}]({image['url']})\n\n" + content + # Insert the content + context["page"] = markdown.markdown(content, extensions=["pymdownx.superfences"]) + + @staticmethod + def _set_group_context(context: dict[str, any], tags: list[Tag]) -> None: + """Update the context with the information for a group of tags.""" + group = tags[0].group context.update({ - "page": markdown.markdown(content, extensions=["pymdownx.superfences"]), - "tag": tag, + "categories": {}, + "pages": utils.get_tag_category(tags, collapse_groups=False), + "page_title": group, + "icon": "fab fa-tags", + "is_tag_listing": True, + "app_name": "content:tag", + "path": f"{group}/", + "tag_url": f"{tags[0].URL_BASE}/{group}" }) - - context["breadcrumb_items"] = [{ - "name": utils.get_category(settings.CONTENT_PAGES_PATH / location)["title"], - "path": str(location) - } for location in [".", "tags"]] - - return context diff --git a/pydis_site/static/css/content/color.css b/pydis_site/static/css/content/color.css new file mode 100644 index 00000000..f4801c28 --- /dev/null +++ b/pydis_site/static/css/content/color.css @@ -0,0 +1,7 @@ +.content .fa-github { + color: black; +} + +.content .fa-github:hover { + color: #7289DA; +} diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css index a3db046c..ec45bfc7 100644 --- a/pydis_site/static/css/content/tag.css +++ b/pydis_site/static/css/content/tag.css @@ -1,11 +1,3 @@ -h1.title a { - color: black; -} - -h1.title a:hover { - color: #7289DA; -} - .content a * { /* This is the original color, but propagated down the chain */ /* which allows for elements inside links, such as codeblocks */ diff --git a/pydis_site/static/js/content/listing.js b/pydis_site/static/js/content/listing.js index 3502cb2a..4b722632 100644 --- a/pydis_site/static/js/content/listing.js +++ b/pydis_site/static/js/content/listing.js @@ -4,6 +4,11 @@ function trimTag() { const containers = document.getElementsByClassName("tag-container"); for (const container of containers) { + if (container.textContent.startsWith("Contains the following tags:")) { + // Tag group, no need to trim + continue; + } + // Remove every element after the first two paragraphs while (container.children.length > 2) { container.removeChild(container.lastChild); diff --git a/pydis_site/templates/content/listing.html b/pydis_site/templates/content/listing.html index 098f4237..934b95f6 100644 --- a/pydis_site/templates/content/listing.html +++ b/pydis_site/templates/content/listing.html @@ -2,6 +2,19 @@ {% extends 'content/base.html' %} {% load static %} +{# Show a GitHub button on tag pages #} +{% block title_element %} +{% if is_tag_listing %} + +
+
{{ block.super }}
+
+ +
+
+{% endif %} +{% endblock %} + {% block page_content %} {# Nested Categories #} {% for category, data in categories.items %} @@ -23,10 +36,10 @@ - + {{ data.title }} - {% if "tags" in location %} + {% if is_tag_listing %}
{{ data.description | safe }}
{% else %}

{{ data.description }}

diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html index 264f63d0..9bd65744 100644 --- a/pydis_site/templates/content/tag.html +++ b/pydis_site/templates/content/tag.html @@ -3,6 +3,7 @@ {% block head %} {{ block.super }} + {{ tag.name }} {% endblock %} @@ -15,7 +16,3 @@ {% endblock %} - -{% block page_content %} - {{ block.super }} -{% endblock %} -- cgit v1.2.3 From 9d73247694e3a97b357b506f83493e96ecf2c4de Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 14 Aug 2022 05:43:22 +0200 Subject: Change Hyperlink Color On Hover Signed-off-by: Hassan Abouelela --- pydis_site/static/css/content/tag.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css index ec45bfc7..32a605a8 100644 --- a/pydis_site/static/css/content/tag.css +++ b/pydis_site/static/css/content/tag.css @@ -3,3 +3,7 @@ /* which allows for elements inside links, such as codeblocks */ color: #7289DA; } + +.content a *:hover { + color: black; +} -- cgit v1.2.3 From 5aeffa5ab4dd8b251d2ae742d1a1e2bf3ba461c7 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 14 Aug 2022 07:14:50 +0200 Subject: Update Tests For Tag Groups Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/tests/test_utils.py | 110 +++++++++++++++++++++++----- pydis_site/apps/content/tests/test_views.py | 68 ++++++++++++++++- pydis_site/apps/content/utils.py | 3 +- 3 files changed, 157 insertions(+), 24 deletions(-) diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index a5d5dcb4..556f633c 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -132,6 +132,7 @@ class TagUtilsTests(TestCase): --- This tag has frontmatter! """), + "This is a grouped tag!", ) # Generate a tar archive with a few tags @@ -146,6 +147,10 @@ class TagUtilsTests(TestCase): (tags_folder / "first_tag.md").write_text(bodies[0]) (tags_folder / "second_tag.md").write_text(bodies[1]) + group_folder = tags_folder / "some_group" + group_folder.mkdir() + (group_folder / "grouped_tag.md").write_text(bodies[2]) + with tarfile.open(tar_folder / "temp.tar", "w") as file: file.add(folder, recursive=True) @@ -158,10 +163,15 @@ class TagUtilsTests(TestCase): ) result = utils.fetch_tags() - self.assertEqual([ + + def sort(_tag: models.Tag) -> str: + return _tag.name + + self.assertEqual(sorted([ models.Tag(name="first_tag", body=bodies[0]), models.Tag(name="second_tag", body=bodies[1]), - ], sorted(result, key=lambda tag: tag.name)) + models.Tag(name="grouped_tag", body=bodies[2], group=group_folder.name), + ], key=sort), sorted(result, key=sort)) def test_get_real_tag(self): """Test that a single tag is returned if it exists.""" @@ -170,30 +180,92 @@ class TagUtilsTests(TestCase): self.assertEqual(tag, result) + def test_get_grouped_tag(self): + """Test fetching a tag from a group.""" + tag = models.Tag.objects.create(name="real-tag", group="real-group") + result = utils.get_tag("real-group/real-tag") + + self.assertEqual(tag, result) + + def test_get_group(self): + """Test fetching a group of tags.""" + included = [ + models.Tag.objects.create(name="tag-1", group="real-group"), + models.Tag.objects.create(name="tag-2", group="real-group"), + models.Tag.objects.create(name="tag-3", group="real-group"), + ] + + models.Tag.objects.create(name="not-included-1") + models.Tag.objects.create(name="not-included-2", group="other-group") + + result = utils.get_tag("real-group") + self.assertListEqual(included, result) + def test_get_tag_404(self): """Test that an error is raised when we fetch a non-existing tag.""" models.Tag.objects.create(name="real-tag") with self.assertRaises(models.Tag.DoesNotExist): utils.get_tag("fake") - def test_category_pages(self): - """Test that the category pages function returns the correct records for tags.""" - models.Tag.objects.create(name="second-tag", body="Normal body") - models.Tag.objects.create(name="first-tag", body="Normal body") - tag_body = {"description": markdown.markdown("Normal body"), "icon": "fas fa-tag"} - + @mock.patch.object(utils, "get_tag_category") + def test_category_pages(self, get_mock: mock.Mock): + """Test that the category pages function calls the correct method for tags.""" + tag = models.Tag.objects.create(name="tag") + get_mock.return_value = tag result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") + self.assertEqual(tag, result) + get_mock.assert_called_once_with(collapse_groups=True) + + def test_get_category_root(self): + """Test that all tags are returned and formatted properly for the tag root page.""" + body = "normal body" + base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + + models.Tag.objects.create(name="tag-1", body=body), + models.Tag.objects.create(name="tag-2", body=body), + models.Tag.objects.create(name="tag-3", body=body), + + models.Tag.objects.create(name="tag-4", body=body, group="tag-group") + models.Tag.objects.create(name="tag-5", body=body, group="tag-group") + + result = utils.get_tag_category(collapse_groups=True) + self.assertDictEqual({ - "first-tag": {**tag_body, "title": "first-tag"}, - "second-tag": {**tag_body, "title": "second-tag"}, + "tag-1": {**base, "title": "tag-1"}, + "tag-2": {**base, "title": "tag-2"}, + "tag-3": {**base, "title": "tag-3"}, + "tag-group": { + "title": "tag-group", + "description": "Contains the following tags: tag-4, tag-5", + "icon": "fas fa-tags" + } }, result) - def test_trimmed_tag_content(self): - """Test a tag with a long body that requires trimming.""" - tag = models.Tag.objects.create(name="long-tag", body="E" * 300) - result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") - self.assertDictEqual({"long-tag": { - "title": "long-tag", - "description": markdown.markdown(tag.body[:100] + "..."), - "icon": "fas fa-tag", - }}, result) + def test_get_category_group(self): + """Test the function for a group root page.""" + body = "normal body" + base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + + included = [ + models.Tag.objects.create(name="tag-1", body=body, group="group"), + models.Tag.objects.create(name="tag-2", body=body, group="group"), + ] + models.Tag.objects.create(name="not-included", body=body) + + result = utils.get_tag_category(included, collapse_groups=False) + self.assertDictEqual({ + "tag-1": {**base, "title": "tag-1"}, + "tag-2": {**base, "title": "tag-2"}, + }, result) + + def test_tag_url(self): + """Test that tag URLs are generated correctly.""" + cases = [ + ({"name": "tag"}, f"{models.Tag.URL_BASE}/tag.md"), + ({"name": "grouped", "group": "abc"}, f"{models.Tag.URL_BASE}/abc/grouped.md"), + ] + + for options, url in cases: + tag = models.Tag(**options) + with self.subTest(tag=tag): + self.assertEqual(url, tag.url) diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index c4d3474e..c5c25be4 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -194,6 +194,23 @@ class TagViewTests(django.test.TestCase): """Set test helpers, then set up fake filesystem.""" super().setUp() + def test_routing(self): + """Test that the correct template is returned for each route.""" + Tag.objects.create(name="example") + Tag.objects.create(name="grouped-tag", group="group-name") + + cases = [ + ("/pages/tags/example/", "content/tag.html"), + ("/pages/tags/group-name/", "content/listing.html"), + ("/pages/tags/group-name/grouped-tag/", "content/tag.html"), + ] + + for url, template in cases: + with self.subTest(url=url): + response = self.client.get(url) + self.assertEqual(200, response.status_code) + self.assertTemplateUsed(response, template) + def test_valid_tag_returns_200(self): """Test that a page is returned for a valid tag.""" Tag.objects.create(name="example", body="This is the tag body.") @@ -207,8 +224,8 @@ class TagViewTests(django.test.TestCase): response = self.client.get("/pages/tags/non-existent/") self.assertEqual(404, response.status_code) - def test_context(self): - """Check that the context contains all the necessary data.""" + def test_context_tag(self): + """Test that the context contains the required data for a tag.""" body = textwrap.dedent(""" --- unused: frontmatter @@ -222,12 +239,55 @@ class TagViewTests(django.test.TestCase): "page_title": "example", "page": markdown.markdown("Tag content here."), "tag": tag, + "breadcrumb_items": [ + {"name": "Pages", "path": "."}, + {"name": "Tags", "path": "tags"}, + ] } for key in expected: self.assertEqual( expected[key], response.context.get(key), f"context.{key} did not match" ) + def test_context_grouped_tag(self): + """ + Test the context for a tag in a group. + + The only difference between this and a regular tag are the breadcrumbs, + so only those are checked. + """ + Tag.objects.create(name="example", body="Body text", group="group-name") + response = self.client.get("/pages/tags/group-name/example/") + self.assertListEqual([ + {"name": "Pages", "path": "."}, + {"name": "Tags", "path": "tags"}, + {"name": "group-name", "path": "tags/group-name"}, + ], response.context.get("breadcrumb_items")) + + def test_group_page(self): + """Test rendering of a group's root page.""" + Tag.objects.create(name="tag-1", body="Body 1", group="group-name") + Tag.objects.create(name="tag-2", body="Body 2", group="group-name") + Tag.objects.create(name="not-included") + + response = self.client.get("/pages/tags/group-name/") + content = response.content.decode("utf-8") + + self.assertInHTML("
group-name
", content) + self.assertInHTML( + f"", + content + ) + self.assertIn(">tag-1", content) + self.assertIn(">tag-2", content) + self.assertNotIn( + ">not-included", + content, + "Tags not in this group shouldn't be rendered." + ) + + self.assertInHTML("

Body 1

", content) + def test_markdown(self): """Test that markdown content is rendered properly.""" body = textwrap.dedent(""" @@ -287,7 +347,7 @@ class TagViewTests(django.test.TestCase): body = filler_before + "`!tags return`" + filler_after Tag.objects.create(name="example", body=body) - other_url = reverse("content:tag", kwargs={"name": "return"}) + other_url = reverse("content:tag", kwargs={"location": "return"}) response = self.client.get("/pages/tags/example/") self.assertEqual( markdown.markdown(filler_before + f"[`!tags return`]({other_url})" + filler_after), @@ -304,7 +364,7 @@ class TagViewTests(django.test.TestCase): content = response.content.decode("utf-8") self.assertTemplateUsed(response, "content/listing.html") - self.assertInHTML('

Tags

', content) + self.assertInHTML('
Tags
', content) for tag_number in range(1, 4): self.assertIn(f"tag-{tag_number}", content) diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index da6a024d..11100ba5 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -48,7 +48,7 @@ def get_tags_static() -> list[Tag]: This will return a cached value, so it should only be used for static builds. """ tags = fetch_tags() - for tag in tags[3:5]: + for tag in tags[3:5]: # pragma: no cover tag.group = "very-cool-group" return tags @@ -190,6 +190,7 @@ def get_tag_category( # Flatten group description into a single string for group in groups.values(): + # If the following string is updated, make sure to update it in the frontend JS as well group["description"] = "Contains the following tags: " + ", ".join(group["description"]) data.append(group) -- cgit v1.2.3 From 25db8f564c0f5c473b165ccab14413ca4471ac7d Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 14 Aug 2022 07:20:34 +0200 Subject: Explicitly Specify Infraction Time In Tests The infraction tests checked that the route returned infractions in the correct order, which is based on insertion time. This can be fragile however, since the insertion time can be very close (or identical) during the tests. That became especially more likely with PR #741 (commit 149e67b4) which improved database access speed. This is fixed by explicitly specifying the insertion time, and spacing them out properly. Signed-off-by: Hassan Abouelela --- pydis_site/apps/api/tests/test_infractions.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pydis_site/apps/api/tests/test_infractions.py b/pydis_site/apps/api/tests/test_infractions.py index f1107734..89ee4e23 100644 --- a/pydis_site/apps/api/tests/test_infractions.py +++ b/pydis_site/apps/api/tests/test_infractions.py @@ -56,15 +56,17 @@ class InfractionTests(AuthenticatedAPITestCase): type='ban', reason='He terk my jerb!', hidden=True, + inserted_at=dt(2020, 10, 10, 0, 0, 0, tzinfo=timezone.utc), expires_at=dt(5018, 11, 20, 15, 52, tzinfo=timezone.utc), - active=True + active=True, ) cls.ban_inactive = Infraction.objects.create( user_id=cls.user.id, actor_id=cls.user.id, type='ban', reason='James is an ass, and we won\'t be working with him again.', - active=False + active=False, + inserted_at=dt(2020, 10, 10, 0, 1, 0, tzinfo=timezone.utc), ) cls.mute_permanent = Infraction.objects.create( user_id=cls.user.id, @@ -72,7 +74,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='mute', reason='He has a filthy mouth and I am his soap.', active=True, - expires_at=None + inserted_at=dt(2020, 10, 10, 0, 2, 0, tzinfo=timezone.utc), + expires_at=None, ) cls.superstar_expires_soon = Infraction.objects.create( user_id=cls.user.id, @@ -80,7 +83,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='superstar', reason='This one doesn\'t matter anymore.', active=True, - expires_at=dt.now(timezone.utc) + datetime.timedelta(hours=5) + inserted_at=dt(2020, 10, 10, 0, 3, 0, tzinfo=timezone.utc), + expires_at=dt.now(timezone.utc) + datetime.timedelta(hours=5), ) cls.voiceban_expires_later = Infraction.objects.create( user_id=cls.user.id, @@ -88,7 +92,8 @@ class InfractionTests(AuthenticatedAPITestCase): type='voice_ban', reason='Jet engine mic', active=True, - expires_at=dt.now(timezone.utc) + datetime.timedelta(days=5) + inserted_at=dt(2020, 10, 10, 0, 4, 0, tzinfo=timezone.utc), + expires_at=dt.now(timezone.utc) + datetime.timedelta(days=5), ) def test_list_all(self): -- cgit v1.2.3 From c5e03fbb8f7a4ebc2f75ce9f8099ba9ef1366f1e Mon Sep 17 00:00:00 2001 From: Ibrahim Date: Tue, 16 Aug 2022 20:13:21 +0530 Subject: striked passed date --- pydis_site/templates/events/pages/code-jams/9/_index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydis_site/templates/events/pages/code-jams/9/_index.html b/pydis_site/templates/events/pages/code-jams/9/_index.html index 7c2617d7..ca7c4f90 100644 --- a/pydis_site/templates/events/pages/code-jams/9/_index.html +++ b/pydis_site/templates/events/pages/code-jams/9/_index.html @@ -27,8 +27,8 @@
  • Wednesday, July 6 - Voting for the theme opens
  • Wednesday, July 13 - The Qualifier closes
  • Thursday, July 21 - Code Jam Begins
  • -
  • Sunday, July 31 - Coding portion of the jam ends
  • -
  • Sunday, August 4 - Code Jam submissions are closed
  • +
  • Sunday, July 31 - Coding portion of the jam ends
  • +
  • Sunday, August 4 - Code Jam submissions are closed
  • The Qualifier

    -- cgit v1.2.3 From 5dfe019745b53ceb8ce37f0db937d6e2a302f6d7 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 18:58:29 +0400 Subject: Move GitHub strptime Format To Settings Signed-off-by: Hassan Abouelela --- pydis_site/apps/api/github_utils.py | 4 +--- pydis_site/settings.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pydis_site/apps/api/github_utils.py b/pydis_site/apps/api/github_utils.py index 5d7bcdc3..e9d7347b 100644 --- a/pydis_site/apps/api/github_utils.py +++ b/pydis_site/apps/api/github_utils.py @@ -11,8 +11,6 @@ from pydis_site import settings MAX_RUN_TIME = datetime.timedelta(minutes=10) """The maximum time allowed before an action is declared timed out.""" -ISO_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" -"""The datetime string format GitHub uses.""" class ArtifactProcessingError(Exception): @@ -147,7 +145,7 @@ def authorize(owner: str, repo: str) -> httpx.Client: def check_run_status(run: WorkflowRun) -> str: """Check if the provided run has been completed, otherwise raise an exception.""" - created_at = datetime.datetime.strptime(run.created_at, ISO_FORMAT_STRING) + created_at = datetime.datetime.strptime(run.created_at, settings.GITHUB_TIMESTAMP_FORMAT) run_time = datetime.datetime.utcnow() - created_at if run.status != "completed": diff --git a/pydis_site/settings.py b/pydis_site/settings.py index 315ea737..9fbd0273 100644 --- a/pydis_site/settings.py +++ b/pydis_site/settings.py @@ -38,6 +38,8 @@ GITHUB_API = "https://api.github.com" GITHUB_TOKEN = env("GITHUB_TOKEN") GITHUB_APP_ID = env("GITHUB_APP_ID") GITHUB_APP_KEY = env("GITHUB_APP_KEY") +GITHUB_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +"""The datetime string format GitHub uses.""" if GITHUB_APP_KEY and (key_file := Path(GITHUB_APP_KEY)).is_file(): # Allow the OAuth key to be loaded from a file -- cgit v1.2.3 From f2374900c4c83097c105b56de02ea82d66bd9466 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 21:19:53 +0400 Subject: Unify Tag Migrations & Add Commit Model Signed-off-by: Hassan Abouelela --- .../apps/content/migrations/0001_add_tags.py | 34 ++++++++++++++++++++ pydis_site/apps/content/migrations/0001_initial.py | 23 -------------- .../migrations/0002_remove_tag_url_tag_group.py | 22 ------------- pydis_site/apps/content/models/__init__.py | 4 +-- pydis_site/apps/content/models/tag.py | 36 ++++++++++++++++++++++ 5 files changed, 72 insertions(+), 47 deletions(-) create mode 100644 pydis_site/apps/content/migrations/0001_add_tags.py delete mode 100644 pydis_site/apps/content/migrations/0001_initial.py delete mode 100644 pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py diff --git a/pydis_site/apps/content/migrations/0001_add_tags.py b/pydis_site/apps/content/migrations/0001_add_tags.py new file mode 100644 index 00000000..2e9d8c45 --- /dev/null +++ b/pydis_site/apps/content/migrations/0001_add_tags.py @@ -0,0 +1,34 @@ +# Generated by Django 4.0.6 on 2022-08-16 16:17 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Commit', + fields=[ + ('sha', models.CharField(help_text='The SHA hash of this commit.', max_length=40, primary_key=True, serialize=False)), + ('message', models.TextField(help_text='The commit message.')), + ('date', models.DateTimeField(help_text='The date and time the commit was created.')), + ('author', models.TextField(help_text='The person(s) who created the commit.')), + ], + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), + ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), + ('group', models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True)), + ('body', models.TextField(help_text='The content of the tag.')), + ('last_commit', models.OneToOneField(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')), + ], + ), + ] diff --git a/pydis_site/apps/content/migrations/0001_initial.py b/pydis_site/apps/content/migrations/0001_initial.py deleted file mode 100644 index 15e3fc95..00000000 --- a/pydis_site/apps/content/migrations/0001_initial.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.0.6 on 2022-08-13 00:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Tag', - fields=[ - ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), - ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), - ('body', models.TextField(help_text='The content of the tag.')), - ('url', models.URLField(help_text='The URL to this tag on GitHub.')), - ], - ), - ] diff --git a/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py b/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py deleted file mode 100644 index e59077f0..00000000 --- a/pydis_site/apps/content/migrations/0002_remove_tag_url_tag_group.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 4.0.6 on 2022-08-13 23:48 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('content', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='tag', - name='url', - ), - migrations.AddField( - model_name='tag', - name='group', - field=models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True), - ), - ] diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py index 2718ce94..60007e27 100644 --- a/pydis_site/apps/content/models/__init__.py +++ b/pydis_site/apps/content/models/__init__.py @@ -1,3 +1,3 @@ -from .tag import Tag +from .tag import Commit, Tag -__all__ = ["Tag"] +__all__ = ["Commit", "Tag"] diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py index 01264ff1..1c89fe1e 100644 --- a/pydis_site/apps/content/models/tag.py +++ b/pydis_site/apps/content/models/tag.py @@ -1,6 +1,36 @@ +import json + from django.db import models +class Commit(models.Model): + """A git commit.""" + + URL_BASE = "https://github.com/python-discord/bot/commit/" + + sha = models.CharField( + help_text="The SHA hash of this commit.", + primary_key=True, + max_length=40, + ) + message = models.TextField(help_text="The commit message.") + date = models.DateTimeField(help_text="The date and time the commit was created.") + author = models.TextField(help_text="The person(s) who created the commit.") + + @property + def url(self) -> str: + """The URL to the commit on GitHub.""" + return self.URL_BASE + self.sha + + @property + def format_users(self) -> str: + """Return a nice representation of the user(s)' name and email.""" + authors = [] + for author in json.loads(self.author): + authors.append(f"{author['name']} <{author['email']}>") + return ", ".join(authors) + + class Tag(models.Model): """A tag from the python-discord server.""" @@ -10,6 +40,12 @@ class Tag(models.Model): help_text="The date and time this data was last fetched.", auto_now=True, ) + last_commit = models.OneToOneField( + Commit, + help_text="The commit this file was last touched in.", + null=True, + on_delete=models.CASCADE, + ) name = models.CharField( help_text="The tag's name.", primary_key=True, -- cgit v1.2.3 From 04babac2f281487adcddbf1e92d9d028896e086e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 21:21:59 +0400 Subject: Add Tag Metadata Uses the commit API to obtain tag metadata such as when it was last edited, and by whom. Signed-off-by: Hassan Abouelela --- .../apps/content/migrations/0001_add_tags.py | 5 +- pydis_site/apps/content/models/tag.py | 14 ++- pydis_site/apps/content/utils.py | 124 ++++++++++++++++++--- pydis_site/static/css/content/tag.css | 6 +- pydis_site/templates/content/tag.html | 22 +++- 5 files changed, 146 insertions(+), 25 deletions(-) diff --git a/pydis_site/apps/content/migrations/0001_add_tags.py b/pydis_site/apps/content/migrations/0001_add_tags.py index 2e9d8c45..73525243 100644 --- a/pydis_site/apps/content/migrations/0001_add_tags.py +++ b/pydis_site/apps/content/migrations/0001_add_tags.py @@ -1,4 +1,4 @@ -# Generated by Django 4.0.6 on 2022-08-16 16:17 +# Generated by Django 4.0.6 on 2022-08-16 17:38 import django.db.models.deletion from django.db import migrations, models @@ -25,10 +25,11 @@ class Migration(migrations.Migration): name='Tag', fields=[ ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), + ('sha', models.CharField(help_text="The tag's hash, as calculated by GitHub.", max_length=40)), ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), ('group', models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True)), ('body', models.TextField(help_text='The content of the tag.')), - ('last_commit', models.OneToOneField(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')), + ('last_commit', models.ForeignKey(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')), ], ), ] diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py index 1c89fe1e..3c729768 100644 --- a/pydis_site/apps/content/models/tag.py +++ b/pydis_site/apps/content/models/tag.py @@ -1,3 +1,4 @@ +import collections.abc import json from django.db import models @@ -22,13 +23,10 @@ class Commit(models.Model): """The URL to the commit on GitHub.""" return self.URL_BASE + self.sha - @property - def format_users(self) -> str: + def format_users(self) -> collections.abc.Iterable[str]: """Return a nice representation of the user(s)' name and email.""" - authors = [] for author in json.loads(self.author): - authors.append(f"{author['name']} <{author['email']}>") - return ", ".join(authors) + yield f"{author['name']} <{author['email']}>" class Tag(models.Model): @@ -40,7 +38,11 @@ class Tag(models.Model): help_text="The date and time this data was last fetched.", auto_now=True, ) - last_commit = models.OneToOneField( + sha = models.CharField( + help_text="The tag's hash, as calculated by GitHub.", + max_length=40, + ) + last_commit = models.ForeignKey( Commit, help_text="The commit this file was last touched in.", null=True, diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index 11100ba5..7b078de6 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -1,5 +1,6 @@ import datetime import functools +import json import tarfile import tempfile import typing @@ -15,11 +16,26 @@ from django.utils import timezone from markdown.extensions.toc import TocExtension from pydis_site import settings -from .models import Tag +from .models import Commit, Tag TAG_CACHE_TTL = datetime.timedelta(hours=1) +def github_client(**kwargs) -> httpx.Client: + """Get a client to access the GitHub API with important settings pre-configured.""" + client = httpx.Client( + base_url=settings.GITHUB_API, + follow_redirects=True, + timeout=settings.TIMEOUT_PERIOD, + **kwargs + ) + if settings.GITHUB_TOKEN: # pragma: no cover + if not client.headers.get("Authorization"): + client.headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} + + return client + + def get_category(path: Path) -> dict[str, str]: """Load category information by name from _info.yml.""" if not path.is_dir(): @@ -60,19 +76,31 @@ def fetch_tags() -> list[Tag]: The entire repository is downloaded and extracted locally because getting file content would require one request per file, and can get rate-limited. """ - if settings.GITHUB_TOKEN: # pragma: no cover - headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} - else: - headers = {} + client = github_client() + + # Grab metadata + metadata = client.get("/repos/python-discord/bot/contents/bot/resources") + metadata.raise_for_status() + + hashes = {} + for entry in metadata.json(): + if entry["type"] == "dir": + # Tag group + files = client.get(entry["url"]) + files.raise_for_status() + files = files.json() + else: + files = [entry] - tar_file = httpx.get( - f"{settings.GITHUB_API}/repos/python-discord/bot/tarball", - follow_redirects=True, - timeout=settings.TIMEOUT_PERIOD, - headers=headers, - ) + for file in files: + hashes[file["name"]] = file["sha"] + + # Download the files + tar_file = client.get("/repos/python-discord/bot/tarball") tar_file.raise_for_status() + client.close() + tags = [] with tempfile.TemporaryDirectory() as folder: with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo: @@ -83,20 +111,83 @@ def fetch_tags() -> list[Tag]: repo.extractall(folder, included) for tag_file in Path(folder).rglob("*.md"): + name = tag_file.name group = None if tag_file.parent.name != "tags": # Tags in sub-folders are considered part of a group group = tag_file.parent.name tags.append(Tag( - name=tag_file.name.removesuffix(".md"), + name=name.removesuffix(".md"), + sha=hashes[name], group=group, body=tag_file.read_text(encoding="utf-8"), + last_commit=None, )) return tags +def set_tag_commit(tag: Tag) -> Tag: + """Fetch commit information from the API, and save it for the tag.""" + path = "/bot/resources/tags" + if tag.group: + path += f"/{tag.group}" + path += f"/{tag.name}.md" + + # Fetch and set the commit + with github_client() as client: + data = client.get("/repos/python-discord/bot/commits", params={"path": path}) + data.raise_for_status() + data = data.json()[0] + + commit = data["commit"] + author, committer = commit["author"], commit["committer"] + + date = datetime.datetime.strptime(committer["date"], settings.GITHUB_TIMESTAMP_FORMAT) + date = date.replace(tzinfo=datetime.timezone.utc) + + if author["email"] == committer["email"]: + commit_author = [author] + else: + commit_author = [author, committer] + + commit_obj, _ = Commit.objects.get_or_create( + sha=data["sha"], + message=commit["message"], + date=date, + author=json.dumps(commit_author), + ) + tag.last_commit = commit_obj + tag.save() + + return tag + + +def record_tags(tags: list[Tag]) -> None: + """Sync the database with an updated set of tags.""" + # Remove entries which no longer exist + Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete() + + # Insert/update the tags + for tag in tags: + try: + old_tag = Tag.objects.get(name=tag.name) + except Tag.DoesNotExist: + # The tag is not in the database yet, + # pretend it's previous state is the current state + old_tag = tag + + if old_tag.sha == tag.sha and old_tag.last_commit is not None: + # We still have an up-to-date commit entry + tag.last_commit = old_tag.last_commit + + tag.save() + + # Drop old, unused commits + Commit.objects.filter(tag__isnull=True).delete() + + def get_tags() -> list[Tag]: """Return a list of all tags visible to the application, from the cache or API.""" if settings.STATIC_BUILD: # pragma: no cover @@ -113,9 +204,7 @@ def get_tags() -> list[Tag]: tags = get_tags_static() else: tags = fetch_tags() - Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete() - for tag in tags: - tag.save() + record_tags(tags) return tags else: @@ -127,6 +216,9 @@ def get_tag(path: str) -> typing.Union[Tag, list[Tag]]: """ Return a tag based on the search location. + If certain tag data is out of sync (for instance a commit date is missing), + an extra request will be made to sync the information. + The tag name and group must match. If only one argument is provided in the path, it's assumed to either be a group name, or a no-group tag name. @@ -142,6 +234,8 @@ def get_tag(path: str) -> typing.Union[Tag, list[Tag]]: matches = [] for tag in get_tags(): if tag.name == name and tag.group == group: + if tag.last_commit is None: + set_tag_commit(tag) return tag elif tag.group == name and group is None: matches.append(tag) diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css index 32a605a8..79795f9e 100644 --- a/pydis_site/static/css/content/tag.css +++ b/pydis_site/static/css/content/tag.css @@ -5,5 +5,9 @@ } .content a *:hover { - color: black; + color: dimgray; +} + +span.update-time { + text-decoration: black underline dotted; } diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html index 9bd65744..513009da 100644 --- a/pydis_site/templates/content/tag.html +++ b/pydis_site/templates/content/tag.html @@ -9,10 +9,30 @@ {% endblock %} {% block title_element %} -
    +
    {{ block.super }}
    + + {% endblock %} -- cgit v1.2.3 From 7c240c68e24c0f3bf041522ce21de271cb92c6f3 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 22:08:39 +0400 Subject: Better Split Up Tag Commit Messages Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/models/tag.py | 5 +++++ pydis_site/templates/content/tag.html | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py index 3c729768..c504ce21 100644 --- a/pydis_site/apps/content/models/tag.py +++ b/pydis_site/apps/content/models/tag.py @@ -23,6 +23,11 @@ class Commit(models.Model): """The URL to the commit on GitHub.""" return self.URL_BASE + self.sha + def lines(self) -> collections.abc.Iterable[str]: + """Return each line in the commit message.""" + for line in self.message.split("\n"): + yield line + def format_users(self) -> collections.abc.Iterable[str]: """Return a nice representation of the user(s)' name and email.""" for author in json.loads(self.author): diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html index 513009da..655dd786 100644 --- a/pydis_site/templates/content/tag.html +++ b/pydis_site/templates/content/tag.html @@ -31,7 +31,9 @@ {% endfor %} - + {% for line in tag.last_commit.lines %} + + {% endfor %}
    -- cgit v1.2.3 From 89b0853245fdf5ba7f1f386d7ea7ab1548b538da Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 22:09:14 +0400 Subject: Fix Tag Metadata For Static Builds Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index 7b078de6..e4a24a73 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -128,8 +128,19 @@ def fetch_tags() -> list[Tag]: return tags -def set_tag_commit(tag: Tag) -> Tag: +def set_tag_commit(tag: Tag) -> None: """Fetch commit information from the API, and save it for the tag.""" + if settings.STATIC_BUILD: + # Static builds request every page during build, which can ratelimit it. + # Instead, we return some fake data. + tag.last_commit = Commit( + sha="68da80efc00d9932a209d5cccd8d344cec0f09ea", + message="Initial Commit\n\nTHIS IS FAKE DEMO DATA", + date=datetime.datetime(2018, 2, 3, 12, 20, 26, tzinfo=datetime.timezone.utc), + author=json.dumps([{"name": "Joseph", "email": "joseph@josephbanks.me"}]), + ) + return + path = "/bot/resources/tags" if tag.group: path += f"/{tag.group}" @@ -161,8 +172,6 @@ def set_tag_commit(tag: Tag) -> Tag: tag.last_commit = commit_obj tag.save() - return tag - def record_tags(tags: list[Tag]) -> None: """Sync the database with an updated set of tags.""" -- cgit v1.2.3 From 92a42694b6ad1a29e5a21e0b3e57639528837113 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 16 Aug 2022 23:45:25 +0400 Subject: Fix Tests For Tag Metadata Signed-off-by: Hassan Abouelela --- pydis_site/apps/api/tests/test_github_utils.py | 7 +- pydis_site/apps/content/tests/test_utils.py | 132 +++++++++++++++++++++++-- pydis_site/apps/content/tests/test_views.py | 36 ++++--- pydis_site/apps/content/utils.py | 2 +- 4 files changed, 148 insertions(+), 29 deletions(-) diff --git a/pydis_site/apps/api/tests/test_github_utils.py b/pydis_site/apps/api/tests/test_github_utils.py index f642f689..6e25bc80 100644 --- a/pydis_site/apps/api/tests/test_github_utils.py +++ b/pydis_site/apps/api/tests/test_github_utils.py @@ -11,6 +11,7 @@ import rest_framework.response import rest_framework.test from django.urls import reverse +from pydis_site import settings from .. import github_utils @@ -49,7 +50,7 @@ class CheckRunTests(unittest.TestCase): "head_sha": "sha", "status": "completed", "conclusion": "success", - "created_at": datetime.datetime.utcnow().strftime(github_utils.ISO_FORMAT_STRING), + "created_at": datetime.datetime.utcnow().strftime(settings.GITHUB_TIMESTAMP_FORMAT), "artifacts_url": "url", } @@ -74,7 +75,7 @@ class CheckRunTests(unittest.TestCase): # to guarantee the right conclusion kwargs["created_at"] = ( datetime.datetime.utcnow() - github_utils.MAX_RUN_TIME - datetime.timedelta(minutes=10) - ).strftime(github_utils.ISO_FORMAT_STRING) + ).strftime(settings.GITHUB_TIMESTAMP_FORMAT) with self.assertRaises(github_utils.RunTimeoutError): github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) @@ -178,7 +179,7 @@ class ArtifactFetcherTests(unittest.TestCase): run = github_utils.WorkflowRun( name="action_name", head_sha="action_sha", - created_at=datetime.datetime.now().strftime(github_utils.ISO_FORMAT_STRING), + created_at=datetime.datetime.now().strftime(settings.GITHUB_TIMESTAMP_FORMAT), status="completed", conclusion="success", artifacts_url="artifacts_url" diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index 556f633c..2ef033e4 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -1,3 +1,5 @@ +import datetime +import json import tarfile import tempfile import textwrap @@ -15,6 +17,18 @@ from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) +_time = datetime.datetime(2022, 10, 10, 10, 10, 10, tzinfo=datetime.timezone.utc) +_time_str = _time.strftime(settings.GITHUB_TIMESTAMP_FORMAT) +TEST_COMMIT_KWARGS = { + "sha": "123", + "message": "Hello world\n\nThis is a commit message", + "date": _time, + "author": json.dumps([ + {"name": "Author 1", "email": "mail1@example.com", "date": _time_str}, + {"name": "Author 2", "email": "mail2@example.com", "date": _time_str}, + ]), +} + class GetCategoryTests(MockPagesTestCase): """Tests for the get_category function.""" @@ -109,6 +123,10 @@ class GetPageTests(MockPagesTestCase): class TagUtilsTests(TestCase): """Tests for the tag-related utilities.""" + def setUp(self) -> None: + super().setUp() + self.commit = models.Commit.objects.create(**TEST_COMMIT_KWARGS) + @mock.patch.object(utils, "fetch_tags") def test_static_fetch(self, fetch_mock: mock.Mock): """Test that the static fetch function is only called at most once during static builds.""" @@ -121,9 +139,27 @@ class TagUtilsTests(TestCase): self.assertEqual(tags, result) self.assertEqual(tags, second_result) - @mock.patch("httpx.get") + @mock.patch("httpx.Client.get") def test_mocked_fetch(self, get_mock: mock.Mock): """Test that proper data is returned from fetch, but with a mocked API response.""" + fake_request = httpx.Request("GET", "https://google.com") + + # Metadata requests + returns = [httpx.Response( + request=fake_request, + status_code=200, + json=[ + {"type": "file", "name": "first_tag.md", "sha": "123"}, + {"type": "file", "name": "second_tag.md", "sha": "456"}, + {"type": "dir", "name": "some_group", "sha": "789", "url": "/some_group"}, + ] + ), httpx.Response( + request=fake_request, + status_code=200, + json=[{"type": "file", "name": "grouped_tag.md", "sha": "789123"}] + )] + + # Main content request bodies = ( "This is the first tag!", textwrap.dedent(""" @@ -156,33 +192,36 @@ class TagUtilsTests(TestCase): body = (tar_folder / "temp.tar").read_bytes() - get_mock.return_value = httpx.Response( + returns.append(httpx.Response( status_code=200, content=body, - request=httpx.Request("GET", "https://google.com"), - ) + request=fake_request, + )) + get_mock.side_effect = returns result = utils.fetch_tags() def sort(_tag: models.Tag) -> str: return _tag.name self.assertEqual(sorted([ - models.Tag(name="first_tag", body=bodies[0]), - models.Tag(name="second_tag", body=bodies[1]), - models.Tag(name="grouped_tag", body=bodies[2], group=group_folder.name), + models.Tag(name="first_tag", body=bodies[0], sha="123"), + models.Tag(name="second_tag", body=bodies[1], sha="245"), + models.Tag(name="grouped_tag", body=bodies[2], group=group_folder.name, sha="789123"), ], key=sort), sorted(result, key=sort)) def test_get_real_tag(self): """Test that a single tag is returned if it exists.""" - tag = models.Tag.objects.create(name="real-tag") + tag = models.Tag.objects.create(name="real-tag", last_commit=self.commit) result = utils.get_tag("real-tag") self.assertEqual(tag, result) def test_get_grouped_tag(self): """Test fetching a tag from a group.""" - tag = models.Tag.objects.create(name="real-tag", group="real-group") + tag = models.Tag.objects.create( + name="real-tag", group="real-group", last_commit=self.commit + ) result = utils.get_tag("real-group/real-tag") self.assertEqual(tag, result) @@ -269,3 +308,78 @@ class TagUtilsTests(TestCase): tag = models.Tag(**options) with self.subTest(tag=tag): self.assertEqual(url, tag.url) + + @mock.patch("httpx.Client.get") + def test_get_tag_commit(self, get_mock: mock.Mock): + """Test the get commit function with a normal tag.""" + tag = models.Tag.objects.create(name="example") + + authors = json.loads(self.commit.author) + + get_mock.return_value = httpx.Response( + request=httpx.Request("GET", "https://google.com"), + status_code=200, + json=[{ + "sha": self.commit.sha, + "commit": { + "message": self.commit.message, + "author": authors[0], + "committer": authors[1], + } + }] + ) + + result = utils.get_tag(tag.name) + self.assertEqual(tag, result) + + get_mock.assert_called_once() + call_params = get_mock.call_args[1]["params"] + + self.assertEqual({"path": "/bot/resources/tags/example.md"}, call_params) + self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + + @mock.patch("httpx.Client.get") + def test_get_group_tag_commit(self, get_mock: mock.Mock): + """Test the get commit function with a group tag.""" + tag = models.Tag.objects.create(name="example", group="group-name") + + authors = json.loads(self.commit.author) + authors.pop() + self.commit.author = json.dumps(authors) + self.commit.save() + + get_mock.return_value = httpx.Response( + request=httpx.Request("GET", "https://google.com"), + status_code=200, + json=[{ + "sha": self.commit.sha, + "commit": { + "message": self.commit.message, + "author": authors[0], + "committer": authors[0], + } + }] + ) + + utils.set_tag_commit(tag) + + get_mock.assert_called_once() + call_params = get_mock.call_args[1]["params"] + + self.assertEqual({"path": "/bot/resources/tags/group-name/example.md"}, call_params) + self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + + @mock.patch.object(utils, "set_tag_commit") + def test_exiting_commit(self, set_commit_mock: mock.Mock): + """Test that a commit is saved when the data has not changed.""" + tag = models.Tag.objects.create(name="tag-name", body="old body", last_commit=self.commit) + + # This is only applied to the object, not to the database + tag.last_commit = None + + utils.record_tags([tag]) + self.assertEqual(self.commit, tag.last_commit) + + result = utils.get_tag("tag-name") + self.assertEqual(tag, result) + set_commit_mock.assert_not_called() diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index c5c25be4..658ac2cc 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -8,10 +8,11 @@ from django.http import Http404 from django.test import RequestFactory, SimpleTestCase, override_settings from django.urls import reverse -from pydis_site.apps.content.models import Tag +from pydis_site.apps.content.models import Commit, Tag from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) +from pydis_site.apps.content.tests.test_utils import TEST_COMMIT_KWARGS from pydis_site.apps.content.views import PageOrCategoryView @@ -193,11 +194,12 @@ class TagViewTests(django.test.TestCase): def setUp(self): """Set test helpers, then set up fake filesystem.""" super().setUp() + self.commit = Commit.objects.create(**TEST_COMMIT_KWARGS) def test_routing(self): """Test that the correct template is returned for each route.""" - Tag.objects.create(name="example") - Tag.objects.create(name="grouped-tag", group="group-name") + Tag.objects.create(name="example", last_commit=self.commit) + Tag.objects.create(name="grouped-tag", group="group-name", last_commit=self.commit) cases = [ ("/pages/tags/example/", "content/tag.html"), @@ -213,7 +215,7 @@ class TagViewTests(django.test.TestCase): def test_valid_tag_returns_200(self): """Test that a page is returned for a valid tag.""" - Tag.objects.create(name="example", body="This is the tag body.") + Tag.objects.create(name="example", body="This is the tag body.", last_commit=self.commit) response = self.client.get("/pages/tags/example/") self.assertEqual(200, response.status_code) self.assertIn("This is the tag body", response.content.decode("utf-8")) @@ -233,7 +235,7 @@ class TagViewTests(django.test.TestCase): Tag content here. """) - tag = Tag.objects.create(name="example", body=body) + tag = Tag.objects.create(name="example", body=body, last_commit=self.commit) response = self.client.get("/pages/tags/example/") expected = { "page_title": "example", @@ -256,7 +258,9 @@ class TagViewTests(django.test.TestCase): The only difference between this and a regular tag are the breadcrumbs, so only those are checked. """ - Tag.objects.create(name="example", body="Body text", group="group-name") + Tag.objects.create( + name="example", body="Body text", group="group-name", last_commit=self.commit + ) response = self.client.get("/pages/tags/group-name/example/") self.assertListEqual([ {"name": "Pages", "path": "."}, @@ -266,9 +270,9 @@ class TagViewTests(django.test.TestCase): def test_group_page(self): """Test rendering of a group's root page.""" - Tag.objects.create(name="tag-1", body="Body 1", group="group-name") - Tag.objects.create(name="tag-2", body="Body 2", group="group-name") - Tag.objects.create(name="not-included") + Tag.objects.create(name="tag-1", body="Body 1", group="group-name", last_commit=self.commit) + Tag.objects.create(name="tag-2", body="Body 2", group="group-name", last_commit=self.commit) + Tag.objects.create(name="not-included", last_commit=self.commit) response = self.client.get("/pages/tags/group-name/") content = response.content.decode("utf-8") @@ -298,7 +302,7 @@ class TagViewTests(django.test.TestCase): **This text is in bold** """) - Tag.objects.create(name="example", body=body) + Tag.objects.create(name="example", body=body, last_commit=self.commit) response = self.client.get("/pages/tags/example/") content = response.content.decode("utf-8") @@ -317,7 +321,7 @@ class TagViewTests(django.test.TestCase): Tag body. """) - Tag.objects.create(name="example", body=body) + Tag.objects.create(name="example", body=body, last_commit=self.commit) response = self.client.get("/pages/tags/example/") content = response.content.decode("utf-8") @@ -333,7 +337,7 @@ class TagViewTests(django.test.TestCase): --- """) - Tag.objects.create(name="example", body=body) + Tag.objects.create(name="example", body=body, last_commit=self.commit) response = self.client.get("/pages/tags/example/") self.assertEqual( "Embed title", @@ -345,7 +349,7 @@ class TagViewTests(django.test.TestCase): """Test hyperlinking of tags works as intended.""" filler_before, filler_after = "empty filler text\n\n", "more\nfiller" body = filler_before + "`!tags return`" + filler_after - Tag.objects.create(name="example", body=body) + Tag.objects.create(name="example", body=body, last_commit=self.commit) other_url = reverse("content:tag", kwargs={"location": "return"}) response = self.client.get("/pages/tags/example/") @@ -356,9 +360,9 @@ class TagViewTests(django.test.TestCase): def test_tag_root_page(self): """Test the root tag page which lists all tags.""" - Tag.objects.create(name="tag-1") - Tag.objects.create(name="tag-2") - Tag.objects.create(name="tag-3") + Tag.objects.create(name="tag-1", last_commit=self.commit) + Tag.objects.create(name="tag-2", last_commit=self.commit) + Tag.objects.create(name="tag-3", last_commit=self.commit) response = self.client.get("/pages/tags/") content = response.content.decode("utf-8") diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index e4a24a73..63f1c41c 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -130,7 +130,7 @@ def fetch_tags() -> list[Tag]: def set_tag_commit(tag: Tag) -> None: """Fetch commit information from the API, and save it for the tag.""" - if settings.STATIC_BUILD: + if settings.STATIC_BUILD: # pragma: no cover # Static builds request every page during build, which can ratelimit it. # Instead, we return some fake data. tag.last_commit = Commit( -- cgit v1.2.3 From c0823236d20e801550fccdbb021d8aabb56d59c0 Mon Sep 17 00:00:00 2001 From: Amrou Bellalouna Date: Thu, 18 Aug 2022 16:58:08 +0100 Subject: add collection of keywords per rule In reference to issue #2108, this commit aims to add an initial set of keywords per rule. These keywords will be later in the "rule" bot command in order to make rule identification easier --- pydis_site/apps/api/views.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/pydis_site/apps/api/views.py b/pydis_site/apps/api/views.py index ad2d948e..f96d6a8d 100644 --- a/pydis_site/apps/api/views.py +++ b/pydis_site/apps/api/views.py @@ -124,35 +124,44 @@ class RulesView(APIView): return Response([ ( - f"Follow the {pydis_coc}." + f"Follow the {pydis_coc}.", + {"coc", "conduct", "code"} ), ( - f"Follow the {discord_community_guidelines} and {discord_tos}." + f"Follow the {discord_community_guidelines} and {discord_tos}.", + {"guidelines", "discord_tos"} ), ( - "Respect staff members and listen to their instructions." + "Respect staff members and listen to their instructions.", + {"staff", "instructions"} ), ( "Use English to the best of your ability. " - "Be polite if someone speaks English imperfectly." + "Be polite if someone speaks English imperfectly.", + {"english", "language"} ), ( "Do not provide or request help on projects that may break laws, " - "breach terms of services, or are malicious or inappropriate." + "breach terms of services, or are malicious or inappropriate.", + {"infraction", "tos", "breach", "malicious", "inappropriate"} ), ( - "Do not post unapproved advertising." + "Do not post unapproved advertising.", + {"ads", "advertising"} ), ( "Keep discussions relevant to the channel topic. " - "Each channel's description tells you the topic." + "Each channel's description tells you the topic.", + {"off-topic", "topic", "relevance"} ), ( "Do not help with ongoing exams. When helping with homework, " - "help people learn how to do the assignment without doing it for them." + "help people learn how to do the assignment without doing it for them.", + {"exams", "assignment", "assignments", "homework"} ), ( - "Do not offer or ask for paid work of any kind." + "Do not offer or ask for paid work of any kind.", + {"work", "money"} ), ]) -- cgit v1.2.3 From c0384c626121684ad4e354aeb817fdbd2741fc4f Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Tue, 23 Aug 2022 01:09:01 +0400 Subject: Improve Tag Commit Naming Signed-off-by: Hassan Abouelela --- pydis_site/apps/content/models/tag.py | 6 +++--- pydis_site/apps/content/urls.py | 2 +- pydis_site/templates/content/tag.html | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py index c504ce21..73d6cb79 100644 --- a/pydis_site/apps/content/models/tag.py +++ b/pydis_site/apps/content/models/tag.py @@ -5,7 +5,7 @@ from django.db import models class Commit(models.Model): - """A git commit.""" + """A git commit from the Python Discord Bot project.""" URL_BASE = "https://github.com/python-discord/bot/commit/" @@ -28,8 +28,8 @@ class Commit(models.Model): for line in self.message.split("\n"): yield line - def format_users(self) -> collections.abc.Iterable[str]: - """Return a nice representation of the user(s)' name and email.""" + def format_authors(self) -> collections.abc.Iterable[str]: + """Return a nice representation of the author(s)' name and email.""" for author in json.loads(self.author): yield f"{author['name']} <{author['email']}>" diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index 03c0015a..163d05bc 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -63,5 +63,5 @@ urlpatterns = [ views.PageOrCategoryView.as_view(), name='page_category', distill_func=get_all_pages - ) + ), ] diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html index 655dd786..fa9e44f5 100644 --- a/pydis_site/templates/content/tag.html +++ b/pydis_site/templates/content/tag.html @@ -27,7 +27,7 @@