Add Tags To Content Listings

Adds bot tags to the content page, as well as a model to go along with
it.

Signed-off-by: Hassan Abouelela <[email protected]>

9 files changed, 177 insertions, 12 deletions

diff --git a/pydis_site/apps/content/migrations/0001_initial.py b/pydis_site/apps/content/migrations/0001_initial.py
new file mode 100644
index 00000000..15e3fc95
--- /dev/null
+++ b/pydis_site/apps/content/migrations/0001_initial.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.0.6 on 2022-08-13 00:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Tag',
+            fields=[
+                ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')),
+                ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)),
+                ('body', models.TextField(help_text='The content of the tag.')),
+                ('url', models.URLField(help_text='The URL to this tag on GitHub.')),
+            ],
+        ),
+    ]
diff --git a/pydis_site/apps/content/migrations/__init__.py b/pydis_site/apps/content/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/pydis_site/apps/content/migrations/__init__.py
diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/pydis_site/apps/content/models/__init__.py
diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py
new file mode 100644
index 00000000..1437b96a
--- /dev/null
+++ b/pydis_site/apps/content/models/tag.py
@@ -0,0 +1,17 @@
+from django.db import models
+
+
+class Tag(models.Model):
+    """A tag from the python-discord server."""
+
+    last_updated = models.DateTimeField(
+        help_text="The date and time this data was last fetched.",
+        auto_now=True,
+    )
+    name = models.CharField(
+        help_text="The tag's name.",
+        primary_key=True,
+        max_length=50,
+    )
+    body = models.TextField(help_text="The content of the tag.")
+    url = models.URLField(help_text="The URL to this tag on GitHub.")
diff --git a/pydis_site/apps/content/resources/tags/_info.yml b/pydis_site/apps/content/resources/tags/_info.yml
new file mode 100644
index 00000000..054125ec
--- /dev/null
+++ b/pydis_site/apps/content/resources/tags/_info.yml
@@ -0,0 +1,3 @@
+title: Tags
+description: Useful snippets that are often used in the server.
+icon: fas fa-tags
diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py
index d3f270ff..a4252284 100644
--- a/pydis_site/apps/content/utils.py
+++ b/pydis_site/apps/content/utils.py
@@ -1,14 +1,26 @@
+import datetime
+import functools
+import tarfile
+import tempfile
+from io import BytesIO
 from pathlib import Path
-from typing import Dict, Tuple
 
 import frontmatter
+import httpx
 import markdown
 import yaml
 from django.http import Http404
+from django.utils import timezone
 from markdown.extensions.toc import TocExtension
 
+from pydis_site import settings
+from .models.tag import Tag
 
-def get_category(path: Path) -> Dict[str, str]:
+TAG_URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags"
+TAG_CACHE_TTL = datetime.timedelta(hours=1)
+
+
+def get_category(path: Path) -> dict[str, str]:
     """Load category information by name from _info.yml."""
     if not path.is_dir():
         raise Http404("Category not found.")
@@ -16,7 +28,7 @@ def get_category(path: Path) -> Dict[str, str]:
     return yaml.safe_load(path.joinpath("_info.yml").read_text(encoding="utf-8"))
 
 
-def get_categories(path: Path) -> Dict[str, Dict]:
+def get_categories(path: Path) -> dict[str, dict]:
     """Get information for all categories."""
     categories = {}
 
@@ -27,8 +39,111 @@ def get_categories(path: Path) -> Dict[str, Dict]:
     return categories
 
 
-def get_category_pages(path: Path) -> Dict[str, Dict]:
[email protected]
+def get_tags_static() -> list[Tag]:
+    """
+    Fetch tag information in static builds.
+
+    This will return a cached value, so it should only be used for static builds.
+    """
+    return fetch_tags()
+
+
+def fetch_tags() -> list[Tag]:
+    """
+    Fetch tag data from the GitHub API.
+
+    The entire repository is downloaded and extracted locally because
+    getting file content would require one request per file, and can get rate-limited.
+    """
+    if settings.GITHUB_TOKEN:
+        headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"}
+    else:
+        headers = {}
+
+    tar_file = httpx.get(
+        f"{settings.GITHUB_API}/repos/python-discord/bot/tarball",
+        follow_redirects=True,
+        timeout=settings.TIMEOUT_PERIOD,
+        headers=headers,
+    )
+    tar_file.raise_for_status()
+
+    tags = []
+    with tempfile.TemporaryDirectory() as folder:
+        with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo:
+            included = []
+            for file in repo.getmembers():
+                if "/bot/resources/tags" in file.path:
+                    included.append(file)
+            repo.extractall(folder, included)
+
+        for tag_file in Path(folder).rglob("*.md"):
+            tags.append(Tag(
+                name=tag_file.name.removesuffix(".md"),
+                body=tag_file.read_text(encoding="utf-8"),
+                url=f"{TAG_URL_BASE}/{tag_file.name}"
+            ))
+
+    return tags
+
+
+def get_tags() -> list[Tag]:
+    """Return a list of all tags visible to the application, from the cache or API."""
+    if settings.STATIC_BUILD:
+        last_update = None
+    else:
+        last_update = (
+            Tag.objects.values_list("last_updated", flat=True)
+            .order_by("last_updated").first()
+        )
+
+    if last_update is None or timezone.now() >= (last_update + TAG_CACHE_TTL):
+        # Stale or empty cache
+        if settings.STATIC_BUILD:
+            tags = get_tags_static()
+        else:
+            tags = fetch_tags()
+            Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete()
+            for tag in tags:
+                tag.save()
+
+        return tags
+    else:
+        # Get tags from database
+        return Tag.objects.all()
+
+
+def get_tag(name: str) -> Tag:
+    """Return a tag by name."""
+    tags = get_tags()
+    for tag in tags:
+        if tag.name == name:
+            return tag
+
+    raise Tag.DoesNotExist()
+
+
+def get_category_pages(path: Path) -> dict[str, dict]:
     """Get all page names and their metadata at a category path."""
+    # Special handling for tags
+    if path == Path(__file__).parent / "resources/tags":
+        tags = {}
+        for tag in get_tags():
+            content = frontmatter.parse(tag.body)[1]
+            if len(content) > 100:
+                # Trim the preview to a maximum of 100 visible characters
+                # This causes some markdown to break, but we ignore that
+                content = content[:100] + "..."
+
+            tags[tag.name] = {
+                "title": tag.name,
+                "description": markdown.markdown(content),
+                "icon": "fas fa-tag"
+            }
+
+        return {name: tags[name] for name in sorted(tags)}
+
     pages = {}
 
     for item in path.glob("*.md"):
@@ -39,7 +154,7 @@ def get_category_pages(path: Path) -> Dict[str, Dict]:
     return pages
 
 
-def get_page(path: Path) -> Tuple[str, Dict]:
+def get_page(path: Path) -> tuple[str, dict]:
     """Get one specific page."""
     if not path.is_file():
         raise Http404("Page not found.")
diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py
index 356eb021..01ce8402 100644
--- a/pydis_site/apps/content/views/page_category.py
+++ b/pydis_site/apps/content/views/page_category.py
@@ -1,4 +1,3 @@
-import typing as t
 from pathlib import Path
 
 import frontmatter
@@ -25,7 +24,7 @@ class PageOrCategoryView(TemplateView):
 
         return super().dispatch(request, *args, **kwargs)
 
-    def get_template_names(self) -> t.List[str]:
+    def get_template_names(self) -> list[str]:
         """Checks if the view uses the page template or listing template."""
         if self.page_path.is_file():
             template_name = "content/page.html"
@@ -36,7 +35,7 @@ class PageOrCategoryView(TemplateView):
 
         return [template_name]
 
-    def get_context_data(self, **kwargs) -> t.Dict[str, t.Any]:
+    def get_context_data(self, **kwargs) -> dict[str, any]:
         """Assign proper context variables based on what resource user requests."""
         context = super().get_context_data(**kwargs)
 
@@ -73,7 +72,7 @@ class PageOrCategoryView(TemplateView):
         return context
 
     @staticmethod
-    def _get_page_context(path: Path) -> t.Dict[str, t.Any]:
+    def _get_page_context(path: Path) -> dict[str, any]:
         page, metadata = utils.get_page(path)
         return {
             "page": page,
@@ -84,7 +83,7 @@ class PageOrCategoryView(TemplateView):
         }
 
     @staticmethod
-    def _get_category_context(path: Path) -> t.Dict[str, t.Any]:
+    def _get_category_context(path: Path) -> dict[str, any]:
         category = utils.get_category(path)
         return {
             "categories": utils.get_categories(path),
diff --git a/pydis_site/templates/content/listing.html b/pydis_site/templates/content/listing.html
index ef0ef919..eeb6b5e2 100644
--- a/pydis_site/templates/content/listing.html
+++ b/pydis_site/templates/content/listing.html
@@ -1,6 +1,8 @@
+{# Base navigation screen for resources #}
 {% extends 'content/base.html' %}
 
 {% block page_content %}
+    {# Nested Categories #}
     {% for category, data in categories.items %}
         <div class="box" style="max-width: 800px;">
             <span class="icon is-size-4 is-medium">
@@ -13,6 +15,8 @@
             <p class="is-italic">{{ data.description }}</p>
         </div>
     {% endfor %}
+
+    {# Single Pages #}
     {% for page, data in pages.items %}
         <div class="box" style="max-width: 800px;">
             <span class="icon is-size-4 is-medium">
@@ -21,7 +25,11 @@
             <a href="{% url "content:page_category" location=path|add:page %}">
                 <span class="is-size-4 has-text-weight-bold">{{ data.title }}</span>
             </a>
-            <p class="is-italic">{{ data.description }}</p>
+            {% if "tags" in location %}
+                <p class="is-italic">{{ data.description | safe }}</p>
+            {% else %}
+                <p class="is-italic">{{ data.description }}</p>
+            {% endif %}
         </div>
     {% endfor %}
 {% endblock %}
diff --git a/pydis_site/templates/content/page.html b/pydis_site/templates/content/page.html
index 759286f6..625c01f1 100644
--- a/pydis_site/templates/content/page.html
+++ b/pydis_site/templates/content/page.html
@@ -5,7 +5,7 @@
     <link rel="stylesheet"
       href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.1/styles/atom-one-dark.min.css">
     <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.1/highlight.min.js"></script>
-    <script>hljs.initHighlightingOnLoad();</script>
+    <script>hljs.highlightAll();</script>
 {% endblock %}
 
 {% block page_content %}