diff options
| author | 2022-11-05 10:49:03 +0100 | |
|---|---|---|
| committer | 2022-11-05 10:49:03 +0100 | |
| commit | ec2a0de778ebd33ef0ee4c73bd1bd4b961ceea4c (patch) | |
| tree | 3300784e4d4d9eefcc76b62d54dcf27ac7518ea5 /pydis_site/apps | |
| parent | Merge pull request #790 from python-discord/mbaruh-bump-psql (diff) | |
| parent | Merge branch 'main' into bot-tags (diff) | |
Merge pull request #763 from python-discord/bot-tags
List Bot Tags On Site
Diffstat (limited to 'pydis_site/apps')
| -rw-r--r-- | pydis_site/apps/api/github_utils.py | 4 | ||||
| -rw-r--r-- | pydis_site/apps/api/tests/test_github_utils.py | 7 | ||||
| -rw-r--r-- | pydis_site/apps/content/migrations/0001_add_tags.py | 35 | ||||
| -rw-r--r-- | pydis_site/apps/content/migrations/__init__.py | 0 | ||||
| -rw-r--r-- | pydis_site/apps/content/models/__init__.py | 3 | ||||
| -rw-r--r-- | pydis_site/apps/content/models/tag.py | 80 | ||||
| -rw-r--r-- | pydis_site/apps/content/resources/tags/_info.yml | 3 | ||||
| -rw-r--r-- | pydis_site/apps/content/tests/test_utils.py | 289 | ||||
| -rw-r--r-- | pydis_site/apps/content/tests/test_views.py | 220 | ||||
| -rw-r--r-- | pydis_site/apps/content/urls.py | 27 | ||||
| -rw-r--r-- | pydis_site/apps/content/utils.py | 282 | ||||
| -rw-r--r-- | pydis_site/apps/content/views/__init__.py | 3 | ||||
| -rw-r--r-- | pydis_site/apps/content/views/page_category.py | 14 | ||||
| -rw-r--r-- | pydis_site/apps/content/views/tags.py | 124 | ||||
| -rw-r--r-- | pydis_site/apps/home/views/home.py | 8 | ||||
| -rw-r--r-- | pydis_site/apps/redirect/urls.py | 2 | 
16 files changed, 1074 insertions, 27 deletions
| diff --git a/pydis_site/apps/api/github_utils.py b/pydis_site/apps/api/github_utils.py index 986c64e1..44c571c3 100644 --- a/pydis_site/apps/api/github_utils.py +++ b/pydis_site/apps/api/github_utils.py @@ -11,8 +11,6 @@ from pydis_site import settings  MAX_RUN_TIME = datetime.timedelta(minutes=10)  """The maximum time allowed before an action is declared timed out.""" -ISO_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" -"""The datetime string format GitHub uses."""  class ArtifactProcessingError(Exception): @@ -147,7 +145,7 @@ def authorize(owner: str, repo: str) -> httpx.Client:  def check_run_status(run: WorkflowRun) -> str:      """Check if the provided run has been completed, otherwise raise an exception.""" -    created_at = datetime.datetime.strptime(run.created_at, ISO_FORMAT_STRING) +    created_at = datetime.datetime.strptime(run.created_at, settings.GITHUB_TIMESTAMP_FORMAT)      run_time = datetime.datetime.utcnow() - created_at      if run.status != "completed": diff --git a/pydis_site/apps/api/tests/test_github_utils.py b/pydis_site/apps/api/tests/test_github_utils.py index 2eaf48d9..95bafec0 100644 --- a/pydis_site/apps/api/tests/test_github_utils.py +++ b/pydis_site/apps/api/tests/test_github_utils.py @@ -11,6 +11,7 @@ import rest_framework.response  import rest_framework.test  from django.urls import reverse +from pydis_site import settings  from .. import github_utils @@ -49,7 +50,7 @@ class CheckRunTests(unittest.TestCase):          "head_sha": "sha",          "status": "completed",          "conclusion": "success", -        "created_at": datetime.datetime.utcnow().strftime(github_utils.ISO_FORMAT_STRING), +        "created_at": datetime.datetime.utcnow().strftime(settings.GITHUB_TIMESTAMP_FORMAT),          "artifacts_url": "url",      } @@ -74,7 +75,7 @@ class CheckRunTests(unittest.TestCase):          # to guarantee the right conclusion          kwargs["created_at"] = (              datetime.datetime.utcnow() - github_utils.MAX_RUN_TIME - datetime.timedelta(minutes=10) -        ).strftime(github_utils.ISO_FORMAT_STRING) +        ).strftime(settings.GITHUB_TIMESTAMP_FORMAT)          with self.assertRaises(github_utils.RunTimeoutError):              github_utils.check_run_status(github_utils.WorkflowRun(**kwargs)) @@ -178,7 +179,7 @@ class ArtifactFetcherTests(unittest.TestCase):                  run = github_utils.WorkflowRun(                      name="action_name",                      head_sha="action_sha", -                    created_at=datetime.datetime.now().strftime(github_utils.ISO_FORMAT_STRING), +                    created_at=datetime.datetime.now().strftime(settings.GITHUB_TIMESTAMP_FORMAT),                      status="completed",                      conclusion="success",                      artifacts_url="artifacts_url" diff --git a/pydis_site/apps/content/migrations/0001_add_tags.py b/pydis_site/apps/content/migrations/0001_add_tags.py new file mode 100644 index 00000000..2c31e4c1 --- /dev/null +++ b/pydis_site/apps/content/migrations/0001_add_tags.py @@ -0,0 +1,35 @@ +# Generated by Django 4.0.6 on 2022-08-23 09:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + +    initial = True + +    dependencies = [ +    ] + +    operations = [ +        migrations.CreateModel( +            name='Commit', +            fields=[ +                ('sha', models.CharField(help_text='The SHA hash of this commit.', max_length=40, primary_key=True, serialize=False)), +                ('message', models.TextField(help_text='The commit message.')), +                ('date', models.DateTimeField(help_text='The date and time the commit was created.')), +                ('authors', models.TextField(help_text='The person(s) who created the commit. This is a serialized JSON object. Refer to the GitHub documentation on the commit endpoint (schema/commit.author & schema/commit.committer) for more info. https://docs.github.com/en/rest/commits/commits#get-a-commit')), +            ], +        ), +        migrations.CreateModel( +            name='Tag', +            fields=[ +                ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), +                ('sha', models.CharField(help_text="The tag's hash, as calculated by GitHub.", max_length=40)), +                ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), +                ('group', models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True)), +                ('body', models.TextField(help_text='The content of the tag.')), +                ('last_commit', models.ForeignKey(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')), +            ], +        ), +    ] diff --git a/pydis_site/apps/content/migrations/__init__.py b/pydis_site/apps/content/migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/pydis_site/apps/content/migrations/__init__.py diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py new file mode 100644 index 00000000..60007e27 --- /dev/null +++ b/pydis_site/apps/content/models/__init__.py @@ -0,0 +1,3 @@ +from .tag import Commit, Tag + +__all__ = ["Commit", "Tag"] diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py new file mode 100644 index 00000000..1a20d775 --- /dev/null +++ b/pydis_site/apps/content/models/tag.py @@ -0,0 +1,80 @@ +import collections.abc +import json + +from django.db import models + + +class Commit(models.Model): +    """A git commit from the Python Discord Bot project.""" + +    URL_BASE = "https://github.com/python-discord/bot/commit/" + +    sha = models.CharField( +        help_text="The SHA hash of this commit.", +        primary_key=True, +        max_length=40, +    ) +    message = models.TextField(help_text="The commit message.") +    date = models.DateTimeField(help_text="The date and time the commit was created.") +    authors = models.TextField(help_text=( +        "The person(s) who created the commit. This is a serialized JSON object. " +        "Refer to the GitHub documentation on the commit endpoint " +        "(schema/commit.author & schema/commit.committer) for more info. " +        "https://docs.github.com/en/rest/commits/commits#get-a-commit" +    )) + +    @property +    def url(self) -> str: +        """The URL to the commit on GitHub.""" +        return self.URL_BASE + self.sha + +    def lines(self) -> collections.abc.Iterable[str]: +        """Return each line in the commit message.""" +        for line in self.message.split("\n"): +            yield line + +    def format_authors(self) -> collections.abc.Iterable[str]: +        """Return a nice representation of the author(s)' name and email.""" +        for author in json.loads(self.authors): +            yield f"{author['name']} <{author['email']}>" + + +class Tag(models.Model): +    """A tag from the python-discord bot repository.""" + +    URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" + +    last_updated = models.DateTimeField( +        help_text="The date and time this data was last fetched.", +        auto_now=True, +    ) +    sha = models.CharField( +        help_text="The tag's hash, as calculated by GitHub.", +        max_length=40, +    ) +    last_commit = models.ForeignKey( +        Commit, +        help_text="The commit this file was last touched in.", +        null=True, +        on_delete=models.CASCADE, +    ) +    name = models.CharField( +        help_text="The tag's name.", +        primary_key=True, +        max_length=50, +    ) +    group = models.CharField( +        help_text="The group the tag belongs to.", +        null=True, +        max_length=50, +    ) +    body = models.TextField(help_text="The content of the tag.") + +    @property +    def url(self) -> str: +        """Get the URL of the tag on GitHub.""" +        url = Tag.URL_BASE +        if self.group: +            url += f"/{self.group}" +        url += f"/{self.name}.md" +        return url diff --git a/pydis_site/apps/content/resources/tags/_info.yml b/pydis_site/apps/content/resources/tags/_info.yml new file mode 100644 index 00000000..054125ec --- /dev/null +++ b/pydis_site/apps/content/resources/tags/_info.yml @@ -0,0 +1,3 @@ +title: Tags +description: Useful snippets that are often used in the server. +icon: fas fa-tags diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index be5ea897..462818b5 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -1,12 +1,34 @@ +import datetime +import json +import tarfile +import tempfile +import textwrap  from pathlib import Path +from unittest import mock +import httpx +import markdown  from django.http import Http404 +from django.test import TestCase -from pydis_site.apps.content import utils +from pydis_site import settings +from pydis_site.apps.content import models, utils  from pydis_site.apps.content.tests.helpers import (      BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA  ) +_time = datetime.datetime(2022, 10, 10, 10, 10, 10, tzinfo=datetime.timezone.utc) +_time_str = _time.strftime(settings.GITHUB_TIMESTAMP_FORMAT) +TEST_COMMIT_KWARGS = { +    "sha": "123", +    "message": "Hello world\n\nThis is a commit message", +    "date": _time, +    "authors": json.dumps([ +        {"name": "Author 1", "email": "[email protected]", "date": _time_str}, +        {"name": "Author 2", "email": "[email protected]", "date": _time_str}, +    ]), +} +  class GetCategoryTests(MockPagesTestCase):      """Tests for the get_category function.""" @@ -96,3 +118,268 @@ class GetPageTests(MockPagesTestCase):      def test_get_nonexistent_page_returns_404(self):          with self.assertRaises(Http404):              utils.get_page(Path(BASE_PATH, "invalid")) + + +class TagUtilsTests(TestCase): +    """Tests for the tag-related utilities.""" + +    def setUp(self) -> None: +        super().setUp() +        self.commit = models.Commit.objects.create(**TEST_COMMIT_KWARGS) + +    @mock.patch.object(utils, "fetch_tags") +    def test_static_fetch(self, fetch_mock: mock.Mock): +        """Test that the static fetch function is only called at most once during static builds.""" +        tags = [models.Tag(name="Name", body="body")] +        fetch_mock.return_value = tags +        result = utils.get_tags_static() +        second_result = utils.get_tags_static() + +        fetch_mock.assert_called_once() +        self.assertEqual(tags, result) +        self.assertEqual(tags, second_result) + +    @mock.patch("httpx.Client.get") +    def test_mocked_fetch(self, get_mock: mock.Mock): +        """Test that proper data is returned from fetch, but with a mocked API response.""" +        fake_request = httpx.Request("GET", "https://google.com") + +        # Metadata requests +        returns = [httpx.Response( +            request=fake_request, +            status_code=200, +            json=[ +                {"type": "file", "name": "first_tag.md", "sha": "123"}, +                {"type": "file", "name": "second_tag.md", "sha": "456"}, +                {"type": "dir", "name": "some_group", "sha": "789", "url": "/some_group"}, +            ] +        ), httpx.Response( +            request=fake_request, +            status_code=200, +            json=[{"type": "file", "name": "grouped_tag.md", "sha": "789123"}] +        )] + +        # Main content request +        bodies = ( +            "This is the first tag!", +            textwrap.dedent(""" +                --- +                frontmatter: empty +                --- +                This tag has frontmatter! +            """), +            "This is a grouped tag!", +        ) + +        # Generate a tar archive with a few tags +        with tempfile.TemporaryDirectory() as tar_folder: +            tar_folder = Path(tar_folder) +            with tempfile.TemporaryDirectory() as folder: +                folder = Path(folder) +                (folder / "ignored_file.md").write_text("This is an ignored file.") +                tags_folder = folder / "bot/resources/tags" +                tags_folder.mkdir(parents=True) + +                (tags_folder / "first_tag.md").write_text(bodies[0]) +                (tags_folder / "second_tag.md").write_text(bodies[1]) + +                group_folder = tags_folder / "some_group" +                group_folder.mkdir() +                (group_folder / "grouped_tag.md").write_text(bodies[2]) + +                with tarfile.open(tar_folder / "temp.tar", "w") as file: +                    file.add(folder, recursive=True) + +                body = (tar_folder / "temp.tar").read_bytes() + +        returns.append(httpx.Response( +            status_code=200, +            content=body, +            request=fake_request, +        )) + +        get_mock.side_effect = returns +        result = utils.fetch_tags() + +        def sort(_tag: models.Tag) -> str: +            return _tag.name + +        self.assertEqual(sorted([ +            models.Tag(name="first_tag", body=bodies[0], sha="123"), +            models.Tag(name="second_tag", body=bodies[1], sha="245"), +            models.Tag(name="grouped_tag", body=bodies[2], group=group_folder.name, sha="789123"), +        ], key=sort), sorted(result, key=sort)) + +    def test_get_real_tag(self): +        """Test that a single tag is returned if it exists.""" +        tag = models.Tag.objects.create(name="real-tag", last_commit=self.commit) +        result = utils.get_tag("real-tag") + +        self.assertEqual(tag, result) + +    def test_get_grouped_tag(self): +        """Test fetching a tag from a group.""" +        tag = models.Tag.objects.create( +            name="real-tag", group="real-group", last_commit=self.commit +        ) +        result = utils.get_tag("real-group/real-tag") + +        self.assertEqual(tag, result) + +    def test_get_group(self): +        """Test fetching a group of tags.""" +        included = [ +            models.Tag.objects.create(name="tag-1", group="real-group"), +            models.Tag.objects.create(name="tag-2", group="real-group"), +            models.Tag.objects.create(name="tag-3", group="real-group"), +        ] + +        models.Tag.objects.create(name="not-included-1") +        models.Tag.objects.create(name="not-included-2", group="other-group") + +        result = utils.get_tag("real-group") +        self.assertListEqual(included, result) + +    def test_get_tag_404(self): +        """Test that an error is raised when we fetch a non-existing tag.""" +        models.Tag.objects.create(name="real-tag") +        with self.assertRaises(models.Tag.DoesNotExist): +            utils.get_tag("fake") + +    @mock.patch.object(utils, "get_tag_category") +    def test_category_pages(self, get_mock: mock.Mock): +        """Test that the category pages function calls the correct method for tags.""" +        tag = models.Tag.objects.create(name="tag") +        get_mock.return_value = tag +        result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") +        self.assertEqual(tag, result) +        get_mock.assert_called_once_with(collapse_groups=True) + +    def test_get_category_root(self): +        """Test that all tags are returned and formatted properly for the tag root page.""" +        body = "normal body" +        base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + +        models.Tag.objects.create(name="tag-1", body=body), +        models.Tag.objects.create(name="tag-2", body=body), +        models.Tag.objects.create(name="tag-3", body=body), + +        models.Tag.objects.create(name="tag-4", body=body, group="tag-group") +        models.Tag.objects.create(name="tag-5", body=body, group="tag-group") + +        result = utils.get_tag_category(collapse_groups=True) + +        self.assertDictEqual({ +            "tag-1": {**base, "title": "tag-1"}, +            "tag-2": {**base, "title": "tag-2"}, +            "tag-3": {**base, "title": "tag-3"}, +            "tag-group": { +                "title": "tag-group", +                "description": "Contains the following tags: tag-4, tag-5", +                "icon": "fas fa-tags" +            } +        }, result) + +    def test_get_category_group(self): +        """Test the function for a group root page.""" +        body = "normal body" +        base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + +        included = [ +            models.Tag.objects.create(name="tag-1", body=body, group="group"), +            models.Tag.objects.create(name="tag-2", body=body, group="group"), +        ] +        models.Tag.objects.create(name="not-included", body=body) + +        result = utils.get_tag_category(included, collapse_groups=False) +        self.assertDictEqual({ +            "tag-1": {**base, "title": "tag-1"}, +            "tag-2": {**base, "title": "tag-2"}, +        }, result) + +    def test_tag_url(self): +        """Test that tag URLs are generated correctly.""" +        cases = [ +            ({"name": "tag"}, f"{models.Tag.URL_BASE}/tag.md"), +            ({"name": "grouped", "group": "abc"}, f"{models.Tag.URL_BASE}/abc/grouped.md"), +        ] + +        for options, url in cases: +            tag = models.Tag(**options) +            with self.subTest(tag=tag): +                self.assertEqual(url, tag.url) + +    @mock.patch("httpx.Client.get") +    def test_get_tag_commit(self, get_mock: mock.Mock): +        """Test the get commit function with a normal tag.""" +        tag = models.Tag.objects.create(name="example") + +        authors = json.loads(self.commit.authors) + +        get_mock.return_value = httpx.Response( +            request=httpx.Request("GET", "https://google.com"), +            status_code=200, +            json=[{ +                "sha": self.commit.sha, +                "commit": { +                    "message": self.commit.message, +                    "author": authors[0], +                    "committer": authors[1], +                } +            }] +        ) + +        result = utils.get_tag(tag.name) +        self.assertEqual(tag, result) + +        get_mock.assert_called_once() +        call_params = get_mock.call_args[1]["params"] + +        self.assertEqual({"path": "/bot/resources/tags/example.md"}, call_params) +        self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + +    @mock.patch("httpx.Client.get") +    def test_get_group_tag_commit(self, get_mock: mock.Mock): +        """Test the get commit function with a group tag.""" +        tag = models.Tag.objects.create(name="example", group="group-name") + +        authors = json.loads(self.commit.authors) +        authors.pop() +        self.commit.authors = json.dumps(authors) +        self.commit.save() + +        get_mock.return_value = httpx.Response( +            request=httpx.Request("GET", "https://google.com"), +            status_code=200, +            json=[{ +                "sha": self.commit.sha, +                "commit": { +                    "message": self.commit.message, +                    "author": authors[0], +                    "committer": authors[0], +                } +            }] +        ) + +        utils.set_tag_commit(tag) + +        get_mock.assert_called_once() +        call_params = get_mock.call_args[1]["params"] + +        self.assertEqual({"path": "/bot/resources/tags/group-name/example.md"}, call_params) +        self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + +    @mock.patch.object(utils, "set_tag_commit") +    def test_exiting_commit(self, set_commit_mock: mock.Mock): +        """Test that a commit is saved when the data has not changed.""" +        tag = models.Tag.objects.create(name="tag-name", body="old body", last_commit=self.commit) + +        # This is only applied to the object, not to the database +        tag.last_commit = None + +        utils.record_tags([tag]) +        self.assertEqual(self.commit, tag.last_commit) + +        result = utils.get_tag("tag-name") +        self.assertEqual(tag, result) +        set_commit_mock.assert_not_called() diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index a09d22d8..3ef9bcc4 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -1,12 +1,18 @@ +import textwrap  from pathlib import Path  from unittest import TestCase +import django.test +import markdown  from django.http import Http404  from django.test import RequestFactory, SimpleTestCase, override_settings +from django.urls import reverse +from pydis_site.apps.content.models import Commit, Tag  from pydis_site.apps.content.tests.helpers import (      BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA  ) +from pydis_site.apps.content.tests.test_utils import TEST_COMMIT_KWARGS  from pydis_site.apps.content.views import PageOrCategoryView @@ -180,3 +186,217 @@ class PageOrCategoryViewTests(MockPagesTestCase, SimpleTestCase, TestCase):                  {"name": PARSED_CATEGORY_INFO["title"], "path": Path("category/subcategory")},              ]          ) + + +class TagViewTests(django.test.TestCase): +    """Tests for the TagView class.""" + +    def setUp(self): +        """Set test helpers, then set up fake filesystem.""" +        super().setUp() +        self.commit = Commit.objects.create(**TEST_COMMIT_KWARGS) + +    def test_routing(self): +        """Test that the correct template is returned for each route.""" +        Tag.objects.create(name="example", last_commit=self.commit) +        Tag.objects.create(name="grouped-tag", group="group-name", last_commit=self.commit) + +        cases = [ +            ("/pages/tags/example/", "content/tag.html"), +            ("/pages/tags/group-name/", "content/listing.html"), +            ("/pages/tags/group-name/grouped-tag/", "content/tag.html"), +        ] + +        for url, template in cases: +            with self.subTest(url=url): +                response = self.client.get(url) +                self.assertEqual(200, response.status_code) +                self.assertTemplateUsed(response, template) + +    def test_valid_tag_returns_200(self): +        """Test that a page is returned for a valid tag.""" +        Tag.objects.create(name="example", body="This is the tag body.", last_commit=self.commit) +        response = self.client.get("/pages/tags/example/") +        self.assertEqual(200, response.status_code) +        self.assertIn("This is the tag body", response.content.decode("utf-8")) +        self.assertTemplateUsed(response, "content/tag.html") + +    def test_invalid_tag_404(self): +        """Test that a tag which doesn't exist raises a 404.""" +        response = self.client.get("/pages/tags/non-existent/") +        self.assertEqual(404, response.status_code) + +    def test_context_tag(self): +        """Test that the context contains the required data for a tag.""" +        body = textwrap.dedent(""" +        --- +        unused: frontmatter +        ---- +        Tag content here. +        """) + +        tag = Tag.objects.create(name="example", body=body, last_commit=self.commit) +        response = self.client.get("/pages/tags/example/") +        expected = { +            "page_title": "example", +            "page": markdown.markdown("Tag content here."), +            "tag": tag, +            "breadcrumb_items": [ +                {"name": "Pages", "path": "."}, +                {"name": "Tags", "path": "tags"}, +            ] +        } +        for key in expected: +            self.assertEqual( +                expected[key], response.context.get(key), f"context.{key} did not match" +            ) + +    def test_context_grouped_tag(self): +        """ +        Test the context for a tag in a group. + +        The only difference between this and a regular tag are the breadcrumbs, +        so only those are checked. +        """ +        Tag.objects.create( +            name="example", body="Body text", group="group-name", last_commit=self.commit +        ) +        response = self.client.get("/pages/tags/group-name/example/") +        self.assertListEqual([ +            {"name": "Pages", "path": "."}, +            {"name": "Tags", "path": "tags"}, +            {"name": "group-name", "path": "tags/group-name"}, +        ], response.context.get("breadcrumb_items")) + +    def test_group_page(self): +        """Test rendering of a group's root page.""" +        Tag.objects.create(name="tag-1", body="Body 1", group="group-name", last_commit=self.commit) +        Tag.objects.create(name="tag-2", body="Body 2", group="group-name", last_commit=self.commit) +        Tag.objects.create(name="not-included", last_commit=self.commit) + +        response = self.client.get("/pages/tags/group-name/") +        content = response.content.decode("utf-8") + +        self.assertInHTML("<div class='level-left'>group-name</div>", content) +        self.assertInHTML( +            f"<a class='level-item fab fa-github' href='{Tag.URL_BASE}/group-name'>", +            content +        ) +        self.assertIn(">tag-1</span>", content) +        self.assertIn(">tag-2</span>", content) +        self.assertNotIn( +            ">not-included</span>", +            content, +            "Tags not in this group shouldn't be rendered." +        ) + +        self.assertInHTML("<p>Body 1</p>", content) + +    def test_markdown(self): +        """Test that markdown content is rendered properly.""" +        body = textwrap.dedent(""" +        ```py +        Hello world! +        ``` + +        **This text is in bold** +        """) + +        Tag.objects.create(name="example", body=body, last_commit=self.commit) +        response = self.client.get("/pages/tags/example/") +        content = response.content.decode("utf-8") + +        self.assertInHTML('<code class="language-py">Hello world!</code>', content) +        self.assertInHTML("<strong>This text is in bold</strong>", content) + +    def test_embed(self): +        """Test that an embed from the frontmatter is treated correctly.""" +        body = textwrap.dedent(""" +        --- +        embed: +            title: Embed title +            image: +                url: https://google.com +        --- +        Tag body. +        """) + +        Tag.objects.create(name="example", body=body, last_commit=self.commit) +        response = self.client.get("/pages/tags/example/") +        content = response.content.decode("utf-8") + +        self.assertInHTML('<img alt="Embed title" src="https://google.com"/>', content) +        self.assertInHTML("<p>Tag body.</p>", content) + +    def test_embed_title(self): +        """Test that the page title gets set to the embed title.""" +        body = textwrap.dedent(""" +        --- +        embed: +            title: Embed title +        --- +        """) + +        Tag.objects.create(name="example", body=body, last_commit=self.commit) +        response = self.client.get("/pages/tags/example/") +        self.assertEqual( +            "Embed title", +            response.context.get("page_title"), +            "The page title must match the embed title." +        ) + +    def test_hyperlinked_item(self): +        """Test hyperlinking of tags works as intended.""" +        filler_before, filler_after = "empty filler text\n\n", "more\nfiller" +        body = filler_before + "`!tags return`" + filler_after +        Tag.objects.create(name="example", body=body, last_commit=self.commit) + +        other_url = reverse("content:tag", kwargs={"location": "return"}) +        response = self.client.get("/pages/tags/example/") +        self.assertEqual( +            markdown.markdown(filler_before + f"[`!tags return`]({other_url})" + filler_after), +            response.context.get("page") +        ) + +    def test_hyperlinked_group(self): +        """Test hyperlinking with a group works as intended.""" +        Tag.objects.create( +            name="example", body="!tags group-name grouped-tag", last_commit=self.commit +        ) +        Tag.objects.create(name="grouped-tag", group="group-name") + +        other_url = reverse("content:tag", kwargs={"location": "group-name/grouped-tag"}) +        response = self.client.get("/pages/tags/example/") +        self.assertEqual( +            markdown.markdown(f"[!tags group-name grouped-tag]({other_url})"), +            response.context.get("page") +        ) + +    def test_hyperlinked_extra_text(self): +        """Test hyperlinking when a tag is followed by extra, unrelated text.""" +        Tag.objects.create( +            name="example", body="!tags other unrelated text", last_commit=self.commit +        ) +        Tag.objects.create(name="other") + +        other_url = reverse("content:tag", kwargs={"location": "other"}) +        response = self.client.get("/pages/tags/example/") +        self.assertEqual( +            markdown.markdown(f"[!tags other]({other_url}) unrelated text"), +            response.context.get("page") +        ) + +    def test_tag_root_page(self): +        """Test the root tag page which lists all tags.""" +        Tag.objects.create(name="tag-1", last_commit=self.commit) +        Tag.objects.create(name="tag-2", last_commit=self.commit) +        Tag.objects.create(name="tag-3", last_commit=self.commit) + +        response = self.client.get("/pages/tags/") +        content = response.content.decode("utf-8") + +        self.assertTemplateUsed(response, "content/listing.html") +        self.assertInHTML('<div class="level-left">Tags</div>', content) + +        for tag_number in range(1, 4): +            self.assertIn(f"tag-{tag_number}</span>", content) diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index f8496095..a7695a27 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -3,7 +3,7 @@ from pathlib import Path  from django_distill import distill_path -from . import views +from . import utils, views  app_name = "content" @@ -29,15 +29,38 @@ def __get_all_files(root: Path, folder: typing.Optional[Path] = None) -> list[st      return results -def get_all_pages() -> typing.Iterator[dict[str, str]]: +DISTILL_RETURN = typing.Iterator[dict[str, str]] + + +def get_all_pages() -> DISTILL_RETURN:      """Yield a dict of all page categories."""      for location in __get_all_files(Path("pydis_site", "apps", "content", "resources")):          yield {"location": location} +def get_all_tags() -> DISTILL_RETURN: +    """Return all tag names and groups in static builds.""" +    # We instantiate the set with None here to make filtering it out later easier +    # whether it was added in the loop or not +    groups = {None} +    for tag in utils.get_tags_static(): +        groups.add(tag.group) +        yield {"location": (f"{tag.group}/" if tag.group else "") + tag.name} + +    groups.remove(None) +    for group in groups: +        yield {"location": group} + +  urlpatterns = [      distill_path("", views.PageOrCategoryView.as_view(), name='pages'),      distill_path( +        "tags/<path:location>/", +        views.TagView.as_view(), +        name="tag", +        distill_func=get_all_tags +    ), +    distill_path(          "<path:location>/",          views.PageOrCategoryView.as_view(),          name='page_category', diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index d3f270ff..c12893ef 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -1,14 +1,41 @@ +import datetime +import functools +import json +import tarfile +import tempfile +from io import BytesIO  from pathlib import Path -from typing import Dict, Tuple  import frontmatter +import httpx  import markdown  import yaml  from django.http import Http404 +from django.utils import timezone  from markdown.extensions.toc import TocExtension +from pydis_site import settings +from .models import Commit, Tag -def get_category(path: Path) -> Dict[str, str]: +TAG_CACHE_TTL = datetime.timedelta(hours=1) + + +def github_client(**kwargs) -> httpx.Client: +    """Get a client to access the GitHub API with important settings pre-configured.""" +    client = httpx.Client( +        base_url=settings.GITHUB_API, +        follow_redirects=True, +        timeout=settings.TIMEOUT_PERIOD, +        **kwargs +    ) +    if settings.GITHUB_TOKEN:  # pragma: no cover +        if not client.headers.get("Authorization"): +            client.headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} + +    return client + + +def get_category(path: Path) -> dict[str, str]:      """Load category information by name from _info.yml."""      if not path.is_dir():          raise Http404("Category not found.") @@ -16,7 +43,7 @@ def get_category(path: Path) -> Dict[str, str]:      return yaml.safe_load(path.joinpath("_info.yml").read_text(encoding="utf-8")) -def get_categories(path: Path) -> Dict[str, Dict]: +def get_categories(path: Path) -> dict[str, dict]:      """Get information for all categories."""      categories = {} @@ -27,8 +54,253 @@ def get_categories(path: Path) -> Dict[str, Dict]:      return categories -def get_category_pages(path: Path) -> Dict[str, Dict]: +def get_tags_static() -> list[Tag]: +    """ +    Fetch tag information in static builds. + +    This also includes some fake tags to preview the tag groups feature. +    This will return a cached value, so it should only be used for static builds. +    """ +    tags = fetch_tags() +    for tag in tags[3:5]:  # pragma: no cover +        tag.group = "very-cool-group" +    return tags + + +def fetch_tags() -> list[Tag]: +    """ +    Fetch tag data from the GitHub API. + +    The entire repository is downloaded and extracted locally because +    getting file content would require one request per file, and can get rate-limited. +    """ +    with github_client() as client: +        # Grab metadata +        metadata = client.get("/repos/python-discord/bot/contents/bot/resources") +        metadata.raise_for_status() + +        hashes = {} +        for entry in metadata.json(): +            if entry["type"] == "dir": +                # Tag group +                files = client.get(entry["url"]) +                files.raise_for_status() +                files = files.json() +            else: +                files = [entry] + +            for file in files: +                hashes[file["name"]] = file["sha"] + +        # Download the files +        tar_file = client.get("/repos/python-discord/bot/tarball") +        tar_file.raise_for_status() + +    tags = [] +    with tempfile.TemporaryDirectory() as folder: +        with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo: +            included = [] +            for file in repo.getmembers(): +                if "/bot/resources/tags" in file.path: +                    included.append(file) +            repo.extractall(folder, included) + +        for tag_file in Path(folder).rglob("*.md"): +            name = tag_file.name +            group = None +            if tag_file.parent.name != "tags": +                # Tags in sub-folders are considered part of a group +                group = tag_file.parent.name + +            tags.append(Tag( +                name=name.removesuffix(".md"), +                sha=hashes[name], +                group=group, +                body=tag_file.read_text(encoding="utf-8"), +                last_commit=None, +            )) + +    return tags + + +def set_tag_commit(tag: Tag) -> None: +    """Fetch commit information from the API, and save it for the tag.""" +    if settings.STATIC_BUILD:  # pragma: no cover +        # Static builds request every page during build, which can ratelimit it. +        # Instead, we return some fake data. +        tag.last_commit = Commit( +            sha="68da80efc00d9932a209d5cccd8d344cec0f09ea", +            message="Initial Commit\n\nTHIS IS FAKE DEMO DATA", +            date=datetime.datetime(2018, 2, 3, 12, 20, 26, tzinfo=datetime.timezone.utc), +            authors=json.dumps([{"name": "Joseph", "email": "[email protected]"}]), +        ) +        return + +    path = "/bot/resources/tags" +    if tag.group: +        path += f"/{tag.group}" +    path += f"/{tag.name}.md" + +    # Fetch and set the commit +    with github_client() as client: +        data = client.get("/repos/python-discord/bot/commits", params={"path": path}) +        data.raise_for_status() +        data = data.json()[0] + +    commit = data["commit"] +    author, committer = commit["author"], commit["committer"] + +    date = datetime.datetime.strptime(committer["date"], settings.GITHUB_TIMESTAMP_FORMAT) +    date = date.replace(tzinfo=datetime.timezone.utc) + +    if author["email"] == committer["email"]: +        authors = [author] +    else: +        authors = [author, committer] + +    commit_obj, _ = Commit.objects.get_or_create( +        sha=data["sha"], +        message=commit["message"], +        date=date, +        authors=json.dumps(authors), +    ) +    tag.last_commit = commit_obj +    tag.save() + + +def record_tags(tags: list[Tag]) -> None: +    """Sync the database with an updated set of tags.""" +    # Remove entries which no longer exist +    Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete() + +    # Insert/update the tags +    for new_tag in tags: +        try: +            old_tag = Tag.objects.get(name=new_tag.name) +        except Tag.DoesNotExist: +            # The tag is not in the database yet, +            # pretend it's previous state is the current state +            old_tag = new_tag + +        if old_tag.sha == new_tag.sha and old_tag.last_commit is not None: +            # We still have an up-to-date commit entry +            new_tag.last_commit = old_tag.last_commit + +        new_tag.save() + +    # Drop old, unused commits +    Commit.objects.filter(tag__isnull=True).delete() + + +def get_tags() -> list[Tag]: +    """Return a list of all tags visible to the application, from the cache or API.""" +    if settings.STATIC_BUILD:  # pragma: no cover +        last_update = None +    else: +        last_update = ( +            Tag.objects.values_list("last_updated", flat=True) +            .order_by("last_updated").first() +        ) + +    if last_update is None or timezone.now() >= (last_update + TAG_CACHE_TTL): +        # Stale or empty cache +        if settings.STATIC_BUILD:  # pragma: no cover +            tags = get_tags_static() +        else: +            tags = fetch_tags() +            record_tags(tags) + +        return tags +    else: +        # Get tags from database +        return list(Tag.objects.all()) + + +def get_tag(path: str, *, skip_sync: bool = False) -> Tag | list[Tag]: +    """ +    Return a tag based on the search location. + +    If certain tag data is out of sync (for instance a commit date is missing), +    an extra request will be made to sync the information. + +    The tag name and group must match. If only one argument is provided in the path, +    it's assumed to either be a group name, or a no-group tag name. + +    If it's a group name, a list of tags which belong to it is returned. +    """ +    path = path.split("/") +    if len(path) == 2: +        group, name = path +    else: +        name = path[0] +        group = None + +    matches = [] +    for tag in get_tags(): +        if tag.name == name and tag.group == group: +            if tag.last_commit is None and not skip_sync: +                set_tag_commit(tag) +            return tag +        elif tag.group == name and group is None: +            matches.append(tag) + +    if matches: +        return matches + +    raise Tag.DoesNotExist() + + +def get_tag_category(tags: list[Tag] | None = None, *, collapse_groups: bool) -> dict[str, dict]: +    """ +    Generate context data for `tags`, or all tags if None. + +    If `tags` is None, `get_tag` is used to populate the data. +    If `collapse_groups` is True, tags with parent groups are not included in the list, +    and instead the parent itself is included as a single entry with it's sub-tags +    in the description. +    """ +    if not tags: +        tags = get_tags() + +    data = [] +    groups = {} + +    # Create all the metadata for the tags +    for tag in tags: +        if tag.group is None or not collapse_groups: +            content = frontmatter.parse(tag.body)[1] +            data.append({ +                "title": tag.name, +                "description": markdown.markdown(content, extensions=["pymdownx.superfences"]), +                "icon": "fas fa-tag", +            }) +        else: +            if tag.group not in groups: +                groups[tag.group] = { +                    "title": tag.group, +                    "description": [tag.name], +                    "icon": "fas fa-tags", +                } +            else: +                groups[tag.group]["description"].append(tag.name) + +    # Flatten group description into a single string +    for group in groups.values(): +        # If the following string is updated, make sure to update it in the frontend JS as well +        group["description"] = "Contains the following tags: " + ", ".join(group["description"]) +        data.append(group) + +    # Sort the tags, and return them in the proper format +    return {tag["title"]: tag for tag in sorted(data, key=lambda tag: tag["title"].casefold())} + + +def get_category_pages(path: Path) -> dict[str, dict]:      """Get all page names and their metadata at a category path.""" +    # Special handling for tags +    if path == Path(__file__).parent / "resources/tags": +        return get_tag_category(collapse_groups=True) +      pages = {}      for item in path.glob("*.md"): @@ -39,7 +311,7 @@ def get_category_pages(path: Path) -> Dict[str, Dict]:      return pages -def get_page(path: Path) -> Tuple[str, Dict]: +def get_page(path: Path) -> tuple[str, dict]:      """Get one specific page."""      if not path.is_file():          raise Http404("Page not found.") diff --git a/pydis_site/apps/content/views/__init__.py b/pydis_site/apps/content/views/__init__.py index 70ea1c7a..a969b1dc 100644 --- a/pydis_site/apps/content/views/__init__.py +++ b/pydis_site/apps/content/views/__init__.py @@ -1,3 +1,4 @@  from .page_category import PageOrCategoryView +from .tags import TagView -__all__ = ["PageOrCategoryView"] +__all__ = ["PageOrCategoryView", "TagView"] diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py index 356eb021..062c2bc1 100644 --- a/pydis_site/apps/content/views/page_category.py +++ b/pydis_site/apps/content/views/page_category.py @@ -1,4 +1,3 @@ -import typing as t  from pathlib import Path  import frontmatter @@ -6,7 +5,7 @@ from django.conf import settings  from django.http import Http404, HttpRequest, HttpResponse  from django.views.generic import TemplateView -from pydis_site.apps.content import utils +from pydis_site.apps.content import models, utils  class PageOrCategoryView(TemplateView): @@ -25,7 +24,7 @@ class PageOrCategoryView(TemplateView):          return super().dispatch(request, *args, **kwargs) -    def get_template_names(self) -> t.List[str]: +    def get_template_names(self) -> list[str]:          """Checks if the view uses the page template or listing template."""          if self.page_path.is_file():              template_name = "content/page.html" @@ -36,7 +35,7 @@ class PageOrCategoryView(TemplateView):          return [template_name] -    def get_context_data(self, **kwargs) -> t.Dict[str, t.Any]: +    def get_context_data(self, **kwargs) -> dict[str, any]:          """Assign proper context variables based on what resource user requests."""          context = super().get_context_data(**kwargs) @@ -73,7 +72,7 @@ class PageOrCategoryView(TemplateView):          return context      @staticmethod -    def _get_page_context(path: Path) -> t.Dict[str, t.Any]: +    def _get_page_context(path: Path) -> dict[str, any]:          page, metadata = utils.get_page(path)          return {              "page": page, @@ -84,7 +83,7 @@ class PageOrCategoryView(TemplateView):          }      @staticmethod -    def _get_category_context(path: Path) -> t.Dict[str, t.Any]: +    def _get_category_context(path: Path) -> dict[str, any]:          category = utils.get_category(path)          return {              "categories": utils.get_categories(path), @@ -92,4 +91,7 @@ class PageOrCategoryView(TemplateView):              "page_title": category["title"],              "page_description": category["description"],              "icon": category.get("icon"), +            "app_name": "content:page_category", +            "is_tag_listing": "/resources/tags" in path.as_posix(), +            "tag_url": models.Tag.URL_BASE,          } diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py new file mode 100644 index 00000000..4f4bb5a2 --- /dev/null +++ b/pydis_site/apps/content/views/tags.py @@ -0,0 +1,124 @@ +import re +import typing + +import frontmatter +import markdown +from django.conf import settings +from django.http import Http404 +from django.urls import reverse +from django.views.generic import TemplateView + +from pydis_site.apps.content import utils +from pydis_site.apps.content.models import Tag + +# The following regex tries to parse a tag command +# It'll read up to two words seperated by spaces +# If the command does not include a group, the tag name will be in the `first` group +# If there's a second word after the command, or if there's a tag group, extra logic +# is necessary to determine whether it's a tag with a group, or a tag with text after it +COMMAND_REGEX = re.compile(r"`*!tags? (?P<first>[\w-]+)(?P<second> [\w-]+)?`*") + + +class TagView(TemplateView): +    """Handles tag pages.""" + +    tag: typing.Union[Tag, list[Tag]] +    is_group: bool + +    def setup(self, *args, **kwargs) -> None: +        """Look for a tag, and configure the view.""" +        super().setup(*args, **kwargs) + +        try: +            self.tag = utils.get_tag(kwargs.get("location")) +            self.is_group = isinstance(self.tag, list) +        except Tag.DoesNotExist: +            raise Http404 + +    def get_template_names(self) -> list[str]: +        """Either return the tag page template, or the listing.""" +        if self.is_group: +            template_name = "content/listing.html" +        else: +            template_name = "content/tag.html" + +        return [template_name] + +    def get_context_data(self, **kwargs) -> dict: +        """Get the relevant context for this tag page or group.""" +        context = super().get_context_data(**kwargs) +        context["breadcrumb_items"] = [{ +            "name": utils.get_category(settings.CONTENT_PAGES_PATH / location)["title"], +            "path": location, +        } for location in (".", "tags")] + +        if self.is_group: +            self._set_group_context(context, self.tag) +        else: +            self._set_tag_context(context, self.tag) + +        return context + +    @staticmethod +    def _set_tag_context(context: dict[str, any], tag: Tag) -> None: +        """Update the context with the information for a tag page.""" +        context.update({ +            "page_title": tag.name, +            "tag": tag, +        }) + +        if tag.group: +            # Add group names to the breadcrumbs +            context["breadcrumb_items"].append({ +                "name": tag.group, +                "path": f"tags/{tag.group}", +            }) + +        # Clean up tag body +        body = frontmatter.parse(tag.body) +        content = body[1] + +        # Check for tags which can be hyperlinked +        def sub(match: re.Match) -> str: +            first, second = match.groups() +            location = first +            text, extra = match.group(), "" + +            if second is not None: +                # Possibly a tag group +                try: +                    new_location = f"{first}/{second.strip()}" +                    utils.get_tag(new_location, skip_sync=True) +                    location = new_location +                except Tag.DoesNotExist: +                    # Not a group, remove the second argument from the link +                    extra = text[text.find(second):] +                    text = text[:text.find(second)] + +            link = reverse("content:tag", kwargs={"location": location}) +            return f"[{text}]({link}){extra}" +        content = COMMAND_REGEX.sub(sub, content) + +        # Add support for some embed elements +        if embed := body[0].get("embed"): +            context["page_title"] = embed["title"] +            if image := embed.get("image"): +                content = f"![{embed['title']}]({image['url']})\n\n" + content + +        # Insert the content +        context["page"] = markdown.markdown(content, extensions=["pymdownx.superfences"]) + +    @staticmethod +    def _set_group_context(context: dict[str, any], tags: list[Tag]) -> None: +        """Update the context with the information for a group of tags.""" +        group = tags[0].group +        context.update({ +            "categories": {}, +            "pages": utils.get_tag_category(tags, collapse_groups=False), +            "page_title": group, +            "icon": "fab fa-tags", +            "is_tag_listing": True, +            "app_name": "content:tag", +            "path": f"{group}/", +            "tag_url": f"{tags[0].URL_BASE}/{group}" +        }) diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 9bb1f8fd..8a165682 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -32,9 +32,7 @@ class HomeView(View):      def __init__(self):          """Clean up stale RepositoryMetadata.""" -        self._static_build = settings.env("STATIC_BUILD") - -        if not self._static_build: +        if not settings.STATIC_BUILD:              RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete()          # If no token is defined (for example in local development), then @@ -94,7 +92,7 @@ class HomeView(View):      def _get_repo_data(self) -> List[RepositoryMetadata]:          """Build a list of RepositoryMetadata objects that we can use to populate the front page."""          # First off, load the timestamp of the least recently updated entry. -        if self._static_build: +        if settings.STATIC_BUILD:              last_update = None          else:              last_update = ( @@ -121,7 +119,7 @@ class HomeView(View):                  for api_data in api_repositories.values()              ] -            if settings.env("STATIC_BUILD"): +            if settings.STATIC_BUILD:                  return data              else:                  return RepositoryMetadata.objects.bulk_create(data) diff --git a/pydis_site/apps/redirect/urls.py b/pydis_site/apps/redirect/urls.py index f86fe665..067cccc3 100644 --- a/pydis_site/apps/redirect/urls.py +++ b/pydis_site/apps/redirect/urls.py @@ -32,7 +32,7 @@ class Redirect:  def map_redirect(name: str, data: Redirect) -> list[URLPattern]:      """Return a pattern using the Redirects app, or a static HTML redirect for static builds.""" -    if not settings.env("STATIC_BUILD"): +    if not settings.STATIC_BUILD:          # Normal dynamic redirect          return [path(              data.original_path, | 
