From a908b9aa3a18222f296c0e4bd67d815f48ada5af Mon Sep 17 00:00:00 2001 From: Johannes Christ Date: Wed, 13 Dec 2023 09:07:08 +0100 Subject: Migrate mailing lists to their own API endpoints Add a new model for the bot to store its mailing list state in, as opposed to the current JSON blob in the BotSetting table. Migrate the existing settings from the BotSetting table into the new model. --- .../apps/api/migrations/0093_add_mailing_lists.py | 36 ++++++++ .../migrations/0094_migrate_mailing_listdata.py | 41 +++++++++ pydis_site/apps/api/models/__init__.py | 2 + pydis_site/apps/api/models/bot/__init__.py | 2 + pydis_site/apps/api/models/bot/mailing_list.py | 13 +++ .../apps/api/models/bot/mailing_list_seen_item.py | 29 +++++++ pydis_site/apps/api/serializers.py | 36 ++++++++ pydis_site/apps/api/tests/test_mailing_list.py | 93 +++++++++++++++++++++ pydis_site/apps/api/urls.py | 5 ++ pydis_site/apps/api/viewsets/__init__.py | 7 +- pydis_site/apps/api/viewsets/bot/__init__.py | 10 +-- pydis_site/apps/api/viewsets/bot/mailing_list.py | 97 ++++++++++++++++++++++ 12 files changed, 362 insertions(+), 9 deletions(-) create mode 100644 pydis_site/apps/api/migrations/0093_add_mailing_lists.py create mode 100644 pydis_site/apps/api/migrations/0094_migrate_mailing_listdata.py create mode 100644 pydis_site/apps/api/models/bot/mailing_list.py create mode 100644 pydis_site/apps/api/models/bot/mailing_list_seen_item.py create mode 100644 pydis_site/apps/api/tests/test_mailing_list.py create mode 100644 pydis_site/apps/api/viewsets/bot/mailing_list.py (limited to 'pydis_site/apps') diff --git a/pydis_site/apps/api/migrations/0093_add_mailing_lists.py b/pydis_site/apps/api/migrations/0093_add_mailing_lists.py new file mode 100644 index 00000000..9f210b94 --- /dev/null +++ b/pydis_site/apps/api/migrations/0093_add_mailing_lists.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0 on 2023-12-17 13:31 + +import django.db.models.deletion +import pydis_site.apps.api.models.mixins +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0092_remove_redirect_filter_list'), + ] + + operations = [ + migrations.CreateModel( + name='MailingList', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='A short identifier for the mailing list.', max_length=50, unique=True)), + ], + bases=(pydis_site.apps.api.models.mixins.ModelReprMixin, models.Model), + ), + migrations.CreateModel( + name='MailingListSeenItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hash', models.CharField(help_text='A hash, or similar identifier, of the content that was seen.', max_length=100)), + ('list', models.ForeignKey(help_text='The mailing list from which this seen item originates.', on_delete=django.db.models.deletion.CASCADE, related_name='seen_items', to='api.mailinglist')), + ], + bases=(pydis_site.apps.api.models.mixins.ModelReprMixin, models.Model), + ), + migrations.AddConstraint( + model_name='mailinglistseenitem', + constraint=models.UniqueConstraint(fields=('list', 'hash'), name='unique_list_and_hash'), + ), + ] diff --git a/pydis_site/apps/api/migrations/0094_migrate_mailing_listdata.py b/pydis_site/apps/api/migrations/0094_migrate_mailing_listdata.py new file mode 100644 index 00000000..50598025 --- /dev/null +++ b/pydis_site/apps/api/migrations/0094_migrate_mailing_listdata.py @@ -0,0 +1,41 @@ +# Generated by Django 5.0 on 2023-12-13 07:03 + +from django.db import migrations + + +def migrate_mailing_lists_into_new_model(apps, schema_editor): + """Move the bot's mailing list information from the BotSetting to the new MailingList model.""" + + BotSetting = apps.get_model('api', 'BotSetting') + MailingList = apps.get_model('api', 'MailingList') + MailingListSeenItem = apps.get_model('api', 'MailingListSeenItem') + try: + setting = BotSetting.objects.get(name='news') + except BotSetting.DoesNotExist: + return + + # Field format: + # { + # "pep": [ + # "644", + # "8102", + # ... + for list_name, item_hashes in setting.data.items(): + (mailing_list, _created) = MailingList.objects.get_or_create(name=list_name) + MailingListSeenItem.objects.bulk_create( + MailingListSeenItem(list=mailing_list, hash=item_hash) + for item_hash in item_hashes + ) + + setting.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0093_add_mailing_lists'), + ] + + operations = [ + migrations.RunPython(migrate_mailing_lists_into_new_model, elidable=True) + ] diff --git a/pydis_site/apps/api/models/__init__.py b/pydis_site/apps/api/models/__init__.py index fee4c8d5..5901c978 100644 --- a/pydis_site/apps/api/models/__init__.py +++ b/pydis_site/apps/api/models/__init__.py @@ -7,6 +7,8 @@ from .bot import ( DocumentationLink, DeletedMessage, Infraction, + MailingList, + MailingListSeenItem, Message, MessageDeletionContext, Nomination, diff --git a/pydis_site/apps/api/models/bot/__init__.py b/pydis_site/apps/api/models/bot/__init__.py index 6f09473d..c07a3238 100644 --- a/pydis_site/apps/api/models/bot/__init__.py +++ b/pydis_site/apps/api/models/bot/__init__.py @@ -8,6 +8,8 @@ from .infraction import Infraction from .message import Message from .aoc_completionist_block import AocCompletionistBlock from .aoc_link import AocAccountLink +from .mailing_list import MailingList +from .mailing_list_seen_item import MailingListSeenItem from .message_deletion_context import MessageDeletionContext from .nomination import Nomination, NominationEntry from .off_topic_channel_name import OffTopicChannelName diff --git a/pydis_site/apps/api/models/bot/mailing_list.py b/pydis_site/apps/api/models/bot/mailing_list.py new file mode 100644 index 00000000..eaca8fb5 --- /dev/null +++ b/pydis_site/apps/api/models/bot/mailing_list.py @@ -0,0 +1,13 @@ +from django.db import models + +from pydis_site.apps.api.models.mixins import ModelReprMixin + + +class MailingList(ModelReprMixin, models.Model): + """A mailing list that the bot is following.""" + + name = models.CharField( + max_length=50, + help_text="A short identifier for the mailing list.", + unique=True + ) diff --git a/pydis_site/apps/api/models/bot/mailing_list_seen_item.py b/pydis_site/apps/api/models/bot/mailing_list_seen_item.py new file mode 100644 index 00000000..d91cfbe6 --- /dev/null +++ b/pydis_site/apps/api/models/bot/mailing_list_seen_item.py @@ -0,0 +1,29 @@ +from django.db import models + +from pydis_site.apps.api.models.mixins import ModelReprMixin +from .mailing_list import MailingList + + +class MailingListSeenItem(ModelReprMixin, models.Model): + """An item in a mailing list that the bot has consumed and mirrored elsewhere.""" + + list = models.ForeignKey( + MailingList, + on_delete=models.CASCADE, + related_name='seen_items', + help_text="The mailing list from which this seen item originates." + ) + hash = models.CharField( + max_length=100, + help_text="A hash, or similar identifier, of the content that was seen." + ) + + class Meta: + """Prevent adding the same hash to the same list multiple times.""" + + constraints = ( + models.UniqueConstraint( + fields=('list', 'hash'), + name='unique_list_and_hash', + ), + ) diff --git a/pydis_site/apps/api/serializers.py b/pydis_site/apps/api/serializers.py index 87fd6190..a2dc68f0 100644 --- a/pydis_site/apps/api/serializers.py +++ b/pydis_site/apps/api/serializers.py @@ -26,6 +26,8 @@ from .models import ( Filter, FilterList, Infraction, + MailingList, + MailingListSeenItem, MessageDeletionContext, Nomination, NominationEntry, @@ -741,3 +743,37 @@ class OffensiveMessageSerializer(FrozenFieldsMixin, ModelSerializer): model = OffensiveMessage fields = ('id', 'channel_id', 'delete_date') frozen_fields = ('id', 'channel_id') + + +class MailingListSeenItemListSerializer(ListSerializer): + """A class providing (de-)serialization of `MailingListSeenItem` instances as a list.""" + + def to_representation(self, objects: list[MailingListSeenItem]) -> list[str]: + """Return the hashes of each seen mailing list item.""" + return [obj['hash'] for obj in objects.values('hash')] + + +class MailingListSeenItemSerializer(ModelSerializer): + """A class providing (de-)serialization of `MailingListSeenItem` instances.""" + + class Meta: + """Metadata defined for the Django REST Framework.""" + + model = MailingListSeenItem + # Since this is only exposed on the parent mailing list model, + # we don't need information about the list or even the ID. + fields = ('hash',) + list_serializer_class = MailingListSeenItemListSerializer + + +class MailingListSerializer(FrozenFieldsMixin, ModelSerializer): + """A class providing (de-)serialization of `MailingList` instances.""" + + seen_items = MailingListSeenItemSerializer(many=True, required=False) + + class Meta: + """Metadata defined for the Django REST Framework.""" + + model = MailingList + fields = ('id', 'name', 'seen_items') + frozen_fields = ('name',) diff --git a/pydis_site/apps/api/tests/test_mailing_list.py b/pydis_site/apps/api/tests/test_mailing_list.py new file mode 100644 index 00000000..2d3025be --- /dev/null +++ b/pydis_site/apps/api/tests/test_mailing_list.py @@ -0,0 +1,93 @@ +from django.urls import reverse + +from .base import AuthenticatedAPITestCase +from pydis_site.apps.api.models import MailingList, MailingListSeenItem + + +class NoMailingListTests(AuthenticatedAPITestCase): + def test_create_mailing_list(self): + url = reverse('api:bot:mailinglist-list') + data = {'name': 'lemon-dev'} + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 201) + +class EmptyMailingListTests(AuthenticatedAPITestCase): + @classmethod + def setUpTestData(cls): + cls.list = MailingList.objects.create(name='erlang-dev') + + def test_create_duplicate_mailing_list(self): + url = reverse('api:bot:mailinglist-list') + data = {'name': self.list.name} + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 400) + + def test_get_all_mailing_lists(self): + url = reverse('api:bot:mailinglist-list') + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), [ + {'id': self.list.id, 'name': self.list.name, 'seen_items': []} + ]) + + def test_get_single_mailing_list(self): + url = reverse('api:bot:mailinglist-detail', args=(self.list.name,)) + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), { + 'id': self.list.id, 'name': self.list.name, 'seen_items': [] + }) + + def test_add_seen_item_to_mailing_list(self): + data = 'PEP-123' + url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,)) + response = self.client.post(url, data=data) + + self.assertEqual(response.status_code, 204) + self.list.refresh_from_db() + self.assertEqual(self.list.seen_items.first().hash, data) + + def test_invalid_request_body(self): + data = [ + "Dinoman, such tiny hands", + "He couldn't even ride a bike", + "He couldn't even dance", + "With the girl that he liked", + "He lived in tiny villages", + "And prayed to tiny god", + "He couldn't go to gameshow", + "Cause he could not applaud...", + ] + url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,)) + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'non_field_errors': ["The request body must be a string"] + }) + + +class MailingListWithSeenItemsTests(AuthenticatedAPITestCase): + @classmethod + def setUpTestData(cls): + cls.list = MailingList.objects.create(name='erlang-dev') + cls.seen_item = MailingListSeenItem.objects.create(hash='12345', list=cls.list) + + def test_get_mailing_list(self): + url = reverse('api:bot:mailinglist-detail', args=(self.list.name,)) + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), { + 'id': self.list.id, 'name': self.list.name, 'seen_items': [self.seen_item.hash] + }) + + def test_prevents_duplicate_addition_of_seen_item(self): + url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,)) + response = self.client.post(url, data=self.seen_item.hash) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'non_field_errors': ["Seen item already known."] + }) diff --git a/pydis_site/apps/api/urls.py b/pydis_site/apps/api/urls.py index 80d4edc2..5cda033a 100644 --- a/pydis_site/apps/api/urls.py +++ b/pydis_site/apps/api/urls.py @@ -17,6 +17,7 @@ from .viewsets import ( FilterListViewSet, FilterViewSet, InfractionViewSet, + MailingListViewSet, NominationViewSet, OffTopicChannelNameViewSet, OffensiveMessageViewSet, @@ -67,6 +68,10 @@ bot_router.register( 'infractions', InfractionViewSet ) +bot_router.register( + 'mailing-lists', + MailingListViewSet +) bot_router.register( 'nominations', NominationViewSet diff --git a/pydis_site/apps/api/viewsets/__init__.py b/pydis_site/apps/api/viewsets/__init__.py index 1dae9be1..a28fa8e3 100644 --- a/pydis_site/apps/api/viewsets/__init__.py +++ b/pydis_site/apps/api/viewsets/__init__.py @@ -1,17 +1,18 @@ # flake8: noqa from .bot import ( + AocAccountLinkViewSet, + AocCompletionistBlockViewSet, BotSettingViewSet, BumpedThreadViewSet, DeletedMessageViewSet, DocumentationLinkViewSet, FilterListViewSet, - InfractionViewSet, FilterListViewSet, FilterViewSet, + InfractionViewSet, + MailingListViewSet, NominationViewSet, OffensiveMessageViewSet, - AocAccountLinkViewSet, - AocCompletionistBlockViewSet, OffTopicChannelNameViewSet, ReminderViewSet, RoleViewSet, diff --git a/pydis_site/apps/api/viewsets/bot/__init__.py b/pydis_site/apps/api/viewsets/bot/__init__.py index 33b65009..bb26cb11 100644 --- a/pydis_site/apps/api/viewsets/bot/__init__.py +++ b/pydis_site/apps/api/viewsets/bot/__init__.py @@ -1,18 +1,16 @@ # flake8: noqa -from .filters import ( - FilterListViewSet, - FilterViewSet -) +from .aoc_completionist_block import AocCompletionistBlockViewSet +from .aoc_link import AocAccountLinkViewSet from .bot_setting import BotSettingViewSet from .bumped_thread import BumpedThreadViewSet from .deleted_message import DeletedMessageViewSet from .documentation_link import DocumentationLinkViewSet +from .filters import FilterListViewSet, FilterViewSet from .infraction import InfractionViewSet +from .mailing_list import MailingListViewSet from .nomination import NominationViewSet from .off_topic_channel_name import OffTopicChannelNameViewSet from .offensive_message import OffensiveMessageViewSet -from .aoc_link import AocAccountLinkViewSet -from .aoc_completionist_block import AocCompletionistBlockViewSet from .reminder import ReminderViewSet from .role import RoleViewSet from .user import UserViewSet diff --git a/pydis_site/apps/api/viewsets/bot/mailing_list.py b/pydis_site/apps/api/viewsets/bot/mailing_list.py new file mode 100644 index 00000000..e46dfd4c --- /dev/null +++ b/pydis_site/apps/api/viewsets/bot/mailing_list.py @@ -0,0 +1,97 @@ +from django.db import IntegrityError +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.exceptions import ParseError +from rest_framework.mixins import CreateModelMixin, ListModelMixin, RetrieveModelMixin +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet + +from pydis_site.apps.api.serializers import MailingListSerializer +from pydis_site.apps.api.models import MailingList, MailingListSeenItem + + +class MailingListViewSet(GenericViewSet, CreateModelMixin, ListModelMixin, RetrieveModelMixin): + """ + View providing management and updates of mailing lists and their seen items. + + ## Routes + + ### GET /bot/mailing-lists + Returns all the mailing lists and their seen items. + + #### Response format + >>> [ + ... { + ... 'id': 1, + ... 'name': 'python-dev', + ... 'seen_items': [ + ... 'd81gg90290la8', + ... ... + ... ] + ... }, + ... ... + ... ] + + ### POST /bot/mailing-lists + Create a new mailing list. + + #### Request format + >>> { + ... 'name': str + ... } + + #### Status codes + - 201: when the mailing list was created successfully + - 400: if the request data was invalid + + ### GET /bot/mailing-lists/ + Retrieve a single mailing list and its seen items. + + #### Response format + >>> { + ... 'id': 1, + ... 'name': 'python-dev', + ... 'seen_items': [ + ... 'd81gg90290la8', + ... ... + ... ] + ... } + + ### POST /bot/mailing-lists//seen-items + Add a single seen item to the given mailing list. The request body should + be the hash of the seen item to add, as a plain string. + + #### Request body + >>> str + + #### Response format + Empty response. + + #### Status codes + - 204: on successful creation of the seen item + - 400: if the request data was invalid + - 404: when the mailing list with the given name could not be found + """ + + lookup_field = 'name' + serializer_class = MailingListSerializer + queryset = MailingList.objects.prefetch_related('seen_items') + + @action(detail=True, methods=["POST"], + name="Add a seen item for a mailing list", url_name='seen-items', url_path='seen-items') + def add_seen_item(self, request: Request, name: str) -> Response: + """Add a single seen item to the given mailing list.""" + if not isinstance(request.data, str): + raise ParseError(detail={'non_field_errors': ["The request body must be a string"]}) + + list_ = self.get_object() + seen_item = MailingListSeenItem(list=list_, hash=request.data) + try: + seen_item.save() + except IntegrityError as err: + if err.__cause__.diag.constraint_name == 'unique_list_and_hash': + raise ParseError(detail={'non_field_errors': ["Seen item already known."]}) + raise # pragma: no cover + + return Response(status=status.HTTP_204_NO_CONTENT) -- cgit v1.2.3