Add Tag Metadata

Uses the commit API to obtain tag metadata such as when it was last
edited, and by whom.

Signed-off-by: Hassan Abouelela <[email protected]>

5 files changed, 146 insertions, 25 deletions

diff --git a/pydis_site/apps/content/migrations/0001_add_tags.py b/pydis_site/apps/content/migrations/0001_add_tags.py
index 2e9d8c45..73525243 100644
--- a/pydis_site/apps/content/migrations/0001_add_tags.py
+++ b/pydis_site/apps/content/migrations/0001_add_tags.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.0.6 on 2022-08-16 16:17
+# Generated by Django 4.0.6 on 2022-08-16 17:38
 
 import django.db.models.deletion
 from django.db import migrations, models
@@ -25,10 +25,11 @@ class Migration(migrations.Migration):
             name='Tag',
             fields=[
                 ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')),
+                ('sha', models.CharField(help_text="The tag's hash, as calculated by GitHub.", max_length=40)),
                 ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)),
                 ('group', models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True)),
                 ('body', models.TextField(help_text='The content of the tag.')),
-                ('last_commit', models.OneToOneField(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')),
+                ('last_commit', models.ForeignKey(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')),
             ],
         ),
     ]
diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py
index 1c89fe1e..3c729768 100644
--- a/pydis_site/apps/content/models/tag.py
+++ b/pydis_site/apps/content/models/tag.py
@@ -1,3 +1,4 @@
+import collections.abc
 import json
 
 from django.db import models
@@ -22,13 +23,10 @@ class Commit(models.Model):
         """The URL to the commit on GitHub."""
         return self.URL_BASE + self.sha
 
-    @property
-    def format_users(self) -> str:
+    def format_users(self) -> collections.abc.Iterable[str]:
         """Return a nice representation of the user(s)' name and email."""
-        authors = []
         for author in json.loads(self.author):
-            authors.append(f"{author['name']} <{author['email']}>")
-        return ", ".join(authors)
+            yield f"{author['name']} <{author['email']}>"
 
 
 class Tag(models.Model):
@@ -40,7 +38,11 @@ class Tag(models.Model):
         help_text="The date and time this data was last fetched.",
         auto_now=True,
     )
-    last_commit = models.OneToOneField(
+    sha = models.CharField(
+        help_text="The tag's hash, as calculated by GitHub.",
+        max_length=40,
+    )
+    last_commit = models.ForeignKey(
         Commit,
         help_text="The commit this file was last touched in.",
         null=True,
diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py
index 11100ba5..7b078de6 100644
--- a/pydis_site/apps/content/utils.py
+++ b/pydis_site/apps/content/utils.py
@@ -1,5 +1,6 @@
 import datetime
 import functools
+import json
 import tarfile
 import tempfile
 import typing
@@ -15,11 +16,26 @@ from django.utils import timezone
 from markdown.extensions.toc import TocExtension
 
 from pydis_site import settings
-from .models import Tag
+from .models import Commit, Tag
 
 TAG_CACHE_TTL = datetime.timedelta(hours=1)
 
 
+def github_client(**kwargs) -> httpx.Client:
+    """Get a client to access the GitHub API with important settings pre-configured."""
+    client = httpx.Client(
+        base_url=settings.GITHUB_API,
+        follow_redirects=True,
+        timeout=settings.TIMEOUT_PERIOD,
+        **kwargs
+    )
+    if settings.GITHUB_TOKEN:  # pragma: no cover
+        if not client.headers.get("Authorization"):
+            client.headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"}
+
+    return client
+
+
 def get_category(path: Path) -> dict[str, str]:
     """Load category information by name from _info.yml."""
     if not path.is_dir():
@@ -60,19 +76,31 @@ def fetch_tags() -> list[Tag]:
     The entire repository is downloaded and extracted locally because
     getting file content would require one request per file, and can get rate-limited.
     """
-    if settings.GITHUB_TOKEN:  # pragma: no cover
-        headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"}
-    else:
-        headers = {}
+    client = github_client()
+
+    # Grab metadata
+    metadata = client.get("/repos/python-discord/bot/contents/bot/resources")
+    metadata.raise_for_status()
+
+    hashes = {}
+    for entry in metadata.json():
+        if entry["type"] == "dir":
+            # Tag group
+            files = client.get(entry["url"])
+            files.raise_for_status()
+            files = files.json()
+        else:
+            files = [entry]
 
-    tar_file = httpx.get(
-        f"{settings.GITHUB_API}/repos/python-discord/bot/tarball",
-        follow_redirects=True,
-        timeout=settings.TIMEOUT_PERIOD,
-        headers=headers,
-    )
+        for file in files:
+            hashes[file["name"]] = file["sha"]
+
+    # Download the files
+    tar_file = client.get("/repos/python-discord/bot/tarball")
     tar_file.raise_for_status()
 
+    client.close()
+
     tags = []
     with tempfile.TemporaryDirectory() as folder:
         with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo:
@@ -83,20 +111,83 @@ def fetch_tags() -> list[Tag]:
             repo.extractall(folder, included)
 
         for tag_file in Path(folder).rglob("*.md"):
+            name = tag_file.name
             group = None
             if tag_file.parent.name != "tags":
                 # Tags in sub-folders are considered part of a group
                 group = tag_file.parent.name
 
             tags.append(Tag(
-                name=tag_file.name.removesuffix(".md"),
+                name=name.removesuffix(".md"),
+                sha=hashes[name],
                 group=group,
                 body=tag_file.read_text(encoding="utf-8"),
+                last_commit=None,
             ))
 
     return tags
 
 
+def set_tag_commit(tag: Tag) -> Tag:
+    """Fetch commit information from the API, and save it for the tag."""
+    path = "/bot/resources/tags"
+    if tag.group:
+        path += f"/{tag.group}"
+    path += f"/{tag.name}.md"
+
+    # Fetch and set the commit
+    with github_client() as client:
+        data = client.get("/repos/python-discord/bot/commits", params={"path": path})
+        data.raise_for_status()
+        data = data.json()[0]
+
+    commit = data["commit"]
+    author, committer = commit["author"], commit["committer"]
+
+    date = datetime.datetime.strptime(committer["date"], settings.GITHUB_TIMESTAMP_FORMAT)
+    date = date.replace(tzinfo=datetime.timezone.utc)
+
+    if author["email"] == committer["email"]:
+        commit_author = [author]
+    else:
+        commit_author = [author, committer]
+
+    commit_obj, _ = Commit.objects.get_or_create(
+        sha=data["sha"],
+        message=commit["message"],
+        date=date,
+        author=json.dumps(commit_author),
+    )
+    tag.last_commit = commit_obj
+    tag.save()
+
+    return tag
+
+
+def record_tags(tags: list[Tag]) -> None:
+    """Sync the database with an updated set of tags."""
+    # Remove entries which no longer exist
+    Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete()
+
+    # Insert/update the tags
+    for tag in tags:
+        try:
+            old_tag = Tag.objects.get(name=tag.name)
+        except Tag.DoesNotExist:
+            # The tag is not in the database yet,
+            # pretend it's previous state is the current state
+            old_tag = tag
+
+        if old_tag.sha == tag.sha and old_tag.last_commit is not None:
+            # We still have an up-to-date commit entry
+            tag.last_commit = old_tag.last_commit
+
+        tag.save()
+
+    # Drop old, unused commits
+    Commit.objects.filter(tag__isnull=True).delete()
+
+
 def get_tags() -> list[Tag]:
     """Return a list of all tags visible to the application, from the cache or API."""
     if settings.STATIC_BUILD:  # pragma: no cover
@@ -113,9 +204,7 @@ def get_tags() -> list[Tag]:
             tags = get_tags_static()
         else:
             tags = fetch_tags()
-            Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete()
-            for tag in tags:
-                tag.save()
+            record_tags(tags)
 
         return tags
     else:
@@ -127,6 +216,9 @@ def get_tag(path: str) -> typing.Union[Tag, list[Tag]]:
     """
     Return a tag based on the search location.
 
+    If certain tag data is out of sync (for instance a commit date is missing),
+    an extra request will be made to sync the information.
+
     The tag name and group must match. If only one argument is provided in the path,
     it's assumed to either be a group name, or a no-group tag name.
 
@@ -142,6 +234,8 @@ def get_tag(path: str) -> typing.Union[Tag, list[Tag]]:
     matches = []
     for tag in get_tags():
         if tag.name == name and tag.group == group:
+            if tag.last_commit is None:
+                set_tag_commit(tag)
             return tag
         elif tag.group == name and group is None:
             matches.append(tag)
diff --git a/pydis_site/static/css/content/tag.css b/pydis_site/static/css/content/tag.css
index 32a605a8..79795f9e 100644
--- a/pydis_site/static/css/content/tag.css
+++ b/pydis_site/static/css/content/tag.css
@@ -5,5 +5,9 @@
 }
 
 .content a *:hover {
-    color: black;
+    color: dimgray;
+}
+
+span.update-time {
+    text-decoration: black underline dotted;
 }
diff --git a/pydis_site/templates/content/tag.html b/pydis_site/templates/content/tag.html
index 9bd65744..513009da 100644
--- a/pydis_site/templates/content/tag.html
+++ b/pydis_site/templates/content/tag.html
@@ -9,10 +9,30 @@
 {% endblock %}
 
 {% block title_element %}
-    <div class="level">
+    <div class="level mb-2">
         <div class="level-left">{{ block.super }}</div>
         <div class="level-right">
             <a class="level-item fab fa-github" href="{{ tag.url }}"></a>
         </div>
     </div>
+
+    <div class="dropdown is-size-6 is-hoverable">
+        <div class="dropdown-trigger ">
+            <a aria-haspopup="menu" href="{{ tag.last_commit.url }}">
+                <span class="update-time">
+                    Last Updated: {{ tag.last_commit.date | date:"F j, Y g:i A e" }}
+                </span>
+            </a>
+        </div>
+        <div class="dropdown-menu">
+            <div class="dropdown-content">
+                <div class="dropdown-item">Last edited by:</div>
+                {% for user in tag.last_commit.format_users %}
+                    <div class="dropdown-item">{{ user }}</div>
+                {% endfor %}
+                <div class="dropdown-divider"></div>
+                <div class="dropdown-item">{{ tag.last_commit.message }}</div>
+            </div>
+        </div>
+    </div>
 {% endblock %}