diff options
Diffstat (limited to 'pydis_site/apps')
38 files changed, 1548 insertions, 379 deletions
diff --git a/pydis_site/apps/api/admin.py b/pydis_site/apps/api/admin.py index 5093e605..449e660e 100644 --- a/pydis_site/apps/api/admin.py +++ b/pydis_site/apps/api/admin.py @@ -14,7 +14,6 @@ from .models import ( DeletedMessage, DocumentationLink, Infraction, - LogEntry, MessageDeletionContext, Nomination, OffTopicChannelName, @@ -22,6 +21,7 @@ from .models import ( Role, User ) +from .models.bot.nomination import NominationEntry admin.site.site_header = "Python Discord | Administration" admin.site.site_title = "Python Discord" @@ -120,33 +120,6 @@ class InfractionAdmin(admin.ModelAdmin): return False [email protected](LogEntry) -class LogEntryAdmin(admin.ModelAdmin): - """Allows viewing logs in the Django Admin without allowing edits.""" - - actions = None - list_display = ('timestamp', 'level', 'message') - fieldsets = ( - ('Overview', {'fields': ('timestamp', 'application', 'logger_name')}), - ('Metadata', {'fields': ('level', 'module', 'line')}), - ('Contents', {'fields': ('message',)}) - ) - list_filter = ('level', 'timestamp') - search_fields = ('message',) - - def has_add_permission(self, request: HttpRequest) -> bool: - """Deny manual LogEntry creation.""" - return False - - def has_change_permission(self, *args) -> bool: - """Prevent editing from django admin.""" - return False - - def has_delete_permission(self, request: HttpRequest, obj: Optional[LogEntry] = None) -> bool: - """Deny LogEntry deletion.""" - return False - - @admin.register(DeletedMessage) class DeletedMessageAdmin(admin.ModelAdmin): """Admin formatting for the DeletedMessage model.""" @@ -246,7 +219,7 @@ class NominationActorFilter(admin.SimpleListFilter): def lookups(self, request: HttpRequest, model: NominationAdmin) -> Iterable[Tuple[int, str]]: """Selectable values for viewer to filter by.""" - actor_ids = Nomination.objects.order_by().values_list("actor").distinct() + actor_ids = NominationEntry.objects.order_by().values_list("actor").distinct() actors = User.objects.filter(id__in=actor_ids) return ((a.id, a.username) for a in actors) @@ -254,7 +227,10 @@ class NominationActorFilter(admin.SimpleListFilter): """Query to filter the list of Users against.""" if not self.value(): return - return queryset.filter(actor__id=self.value()) + nomination_ids = NominationEntry.objects.filter( + actor__id=self.value() + ).values_list("nomination_id").distinct() + return queryset.filter(id__in=nomination_ids) @admin.register(Nomination) @@ -264,9 +240,6 @@ class NominationAdmin(admin.ModelAdmin): search_fields = ( "user__name", "user__id", - "actor__name", - "actor__id", - "reason", "end_reason" ) @@ -275,27 +248,25 @@ class NominationAdmin(admin.ModelAdmin): list_display = ( "user", "active", - "reason", - "actor", + "reviewed" ) fields = ( "user", "active", - "actor", - "reason", "inserted_at", "ended_at", - "end_reason" + "end_reason", + "reviewed" ) - # only allow reason fields to be edited. + # only allow end reason field to be edited. readonly_fields = ( "user", "active", - "actor", "inserted_at", - "ended_at" + "ended_at", + "reviewed" ) def has_add_permission(self, *args) -> bool: @@ -303,6 +274,61 @@ class NominationAdmin(admin.ModelAdmin): return False +class NominationEntryActorFilter(admin.SimpleListFilter): + """Actor Filter for NominationEntry Admin list page.""" + + title = "Actor" + parameter_name = "actor" + + def lookups(self, request: HttpRequest, model: NominationAdmin) -> Iterable[Tuple[int, str]]: + """Selectable values for viewer to filter by.""" + actor_ids = NominationEntry.objects.order_by().values_list("actor").distinct() + actors = User.objects.filter(id__in=actor_ids) + return ((a.id, a.username) for a in actors) + + def queryset(self, request: HttpRequest, queryset: QuerySet) -> Optional[QuerySet]: + """Query to filter the list of Users against.""" + if not self.value(): + return + return queryset.filter(actor__id=self.value()) + + [email protected](NominationEntry) +class NominationEntryAdmin(admin.ModelAdmin): + """Admin formatting for the NominationEntry model.""" + + search_fields = ( + "actor__name", + "actor__id", + "reason", + ) + + list_filter = (NominationEntryActorFilter,) + + list_display = ( + "nomination", + "actor", + ) + + fields = ( + "nomination", + "actor", + "reason", + "inserted_at", + ) + + # only allow reason field to be edited + readonly_fields = ( + "nomination", + "actor", + "inserted_at", + ) + + def has_add_permission(self, request: HttpRequest) -> bool: + """Disable adding new nomination entry from admin.""" + return False + + @admin.register(OffTopicChannelName) class OffTopicChannelNameAdmin(admin.ModelAdmin): """Admin formatting for the OffTopicChannelName model.""" diff --git a/pydis_site/apps/api/dblogger.py b/pydis_site/apps/api/dblogger.py deleted file mode 100644 index 4b4e3a9d..00000000 --- a/pydis_site/apps/api/dblogger.py +++ /dev/null @@ -1,22 +0,0 @@ -from logging import LogRecord, StreamHandler - - -class DatabaseLogHandler(StreamHandler): - """Logs entries into the database.""" - - def emit(self, record: LogRecord) -> None: - """Write the given `record` into the database.""" - # This import needs to be deferred due to Django's application - # registry instantiation logic loading this handler before the - # application is ready. - from pydis_site.apps.api.models.log_entry import LogEntry - - entry = LogEntry( - application='site', - logger_name=record.name, - level=record.levelname.lower(), - module=record.module, - line=record.lineno, - message=self.format(record) - ) - entry.save() diff --git a/pydis_site/apps/api/migrations/0064_delete_logentry.py b/pydis_site/apps/api/migrations/0064_delete_logentry.py new file mode 100644 index 00000000..a5f344d1 --- /dev/null +++ b/pydis_site/apps/api/migrations/0064_delete_logentry.py @@ -0,0 +1,16 @@ +# Generated by Django 3.0.9 on 2020-10-03 06:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0063_Allow_blank_or_null_for_nomination_reason'), + ] + + operations = [ + migrations.DeleteModel( + name='LogEntry', + ), + ] diff --git a/pydis_site/apps/api/migrations/0066_merge_20201003_0730.py b/pydis_site/apps/api/migrations/0066_merge_20201003_0730.py new file mode 100644 index 00000000..298416db --- /dev/null +++ b/pydis_site/apps/api/migrations/0066_merge_20201003_0730.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.9 on 2020-10-03 07:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0064_delete_logentry'), + ('api', '0065_auto_20200919_2033'), + ] + + operations = [ + ] diff --git a/pydis_site/apps/api/migrations/0067_add_voice_ban_infraction_type.py b/pydis_site/apps/api/migrations/0067_add_voice_ban_infraction_type.py new file mode 100644 index 00000000..9a940ff4 --- /dev/null +++ b/pydis_site/apps/api/migrations/0067_add_voice_ban_infraction_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.10 on 2020-10-10 16:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0066_merge_20201003_0730'), + ] + + operations = [ + migrations.AlterField( + model_name='infraction', + name='type', + field=models.CharField(choices=[('note', 'Note'), ('warning', 'Warning'), ('watch', 'Watch'), ('mute', 'Mute'), ('kick', 'Kick'), ('ban', 'Ban'), ('superstar', 'Superstar'), ('voice_ban', 'Voice Ban')], help_text='The type of the infraction.', max_length=9), + ), + ] diff --git a/pydis_site/apps/api/migrations/0068_split_nomination_tables.py b/pydis_site/apps/api/migrations/0068_split_nomination_tables.py new file mode 100644 index 00000000..79825ed7 --- /dev/null +++ b/pydis_site/apps/api/migrations/0068_split_nomination_tables.py @@ -0,0 +1,75 @@ +# Generated by Django 3.0.11 on 2021-02-21 15:32 + +from django.apps.registry import Apps +from django.db import backends, migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +import django.db.models.deletion +import pydis_site.apps.api.models.mixins + + +def migrate_nominations(apps: Apps, schema_editor: BaseDatabaseSchemaEditor) -> None: + Nomination = apps.get_model("api", "Nomination") + NominationEntry = apps.get_model("api", "NominationEntry") + + for nomination in Nomination.objects.all(): + nomination_entry = NominationEntry( + nomination=nomination, + actor=nomination.actor, + reason=nomination.reason, + inserted_at=nomination.inserted_at + ) + nomination_entry.save() + + +def unmigrate_nominations(apps: Apps, schema_editor: BaseDatabaseSchemaEditor) -> None: + Nomination = apps.get_model("api", "Nomination") + NominationEntry = apps.get_model("api", "NominationEntry") + + for entry in NominationEntry.objects.all(): + nomination = Nomination.objects.get(pk=entry.nomination.id) + nomination.actor = entry.actor + nomination.reason = entry.reason + nomination.inserted_at = entry.inserted_at + + nomination.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0067_add_voice_ban_infraction_type'), + ] + + operations = [ + migrations.CreateModel( + name='NominationEntry', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('reason', models.TextField(blank=True, help_text='Why the actor nominated this user.', default="")), + ('inserted_at', + models.DateTimeField(auto_now_add=True, help_text='The creation date of this nomination entry.')), + ('actor', models.ForeignKey(help_text='The staff member that nominated this user.', + on_delete=django.db.models.deletion.CASCADE, related_name='nomination_set', + to='api.User')), + ('nomination', models.ForeignKey(help_text='The nomination this entry belongs to.', + on_delete=django.db.models.deletion.CASCADE, to='api.Nomination', + related_name='entries')), + ], + bases=(pydis_site.apps.api.models.mixins.ModelReprMixin, models.Model), + options={'ordering': ('-inserted_at',), 'verbose_name_plural': 'nomination entries'} + ), + migrations.RunPython(migrate_nominations, unmigrate_nominations), + migrations.RemoveField( + model_name='nomination', + name='actor', + ), + migrations.RemoveField( + model_name='nomination', + name='reason', + ), + migrations.AddField( + model_name='nomination', + name='reviewed', + field=models.BooleanField(default=False, help_text='Whether a review was made.'), + ), + ] diff --git a/pydis_site/apps/api/migrations/0069_documentationlink_validators.py b/pydis_site/apps/api/migrations/0069_documentationlink_validators.py new file mode 100644 index 00000000..347c0e1a --- /dev/null +++ b/pydis_site/apps/api/migrations/0069_documentationlink_validators.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.11 on 2021-03-26 18:21 + +import django.core.validators +from django.db import migrations, models +import pydis_site.apps.api.models.bot.documentation_link + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0068_split_nomination_tables'), + ] + + operations = [ + migrations.AlterField( + model_name='documentationlink', + name='base_url', + field=models.URLField(help_text='The base URL from which documentation will be available for this project. Used to generate links to various symbols within this package.', validators=[pydis_site.apps.api.models.bot.documentation_link.ends_with_slash_validator]), + ), + migrations.AlterField( + model_name='documentationlink', + name='package', + field=models.CharField(help_text='The Python package name that this documentation link belongs to.', max_length=50, primary_key=True, serialize=False, validators=[django.core.validators.RegexValidator(message='Package names can only consist of lowercase a-z letters, digits, and underscores.', regex='^[a-z0-9_]+$')]), + ), + ] diff --git a/pydis_site/apps/api/models/__init__.py b/pydis_site/apps/api/models/__init__.py index e3f928e1..fd5bf220 100644 --- a/pydis_site/apps/api/models/__init__.py +++ b/pydis_site/apps/api/models/__init__.py @@ -8,10 +8,10 @@ from .bot import ( Message, MessageDeletionContext, Nomination, + NominationEntry, OffensiveMessage, OffTopicChannelName, Reminder, Role, User ) -from .log_entry import LogEntry diff --git a/pydis_site/apps/api/models/bot/__init__.py b/pydis_site/apps/api/models/bot/__init__.py index 1673b434..ac864de3 100644 --- a/pydis_site/apps/api/models/bot/__init__.py +++ b/pydis_site/apps/api/models/bot/__init__.py @@ -6,7 +6,7 @@ from .documentation_link import DocumentationLink from .infraction import Infraction from .message import Message from .message_deletion_context import MessageDeletionContext -from .nomination import Nomination +from .nomination import Nomination, NominationEntry from .off_topic_channel_name import OffTopicChannelName from .offensive_message import OffensiveMessage from .reminder import Reminder diff --git a/pydis_site/apps/api/models/bot/documentation_link.py b/pydis_site/apps/api/models/bot/documentation_link.py index 2a0ce751..3dcc71fc 100644 --- a/pydis_site/apps/api/models/bot/documentation_link.py +++ b/pydis_site/apps/api/models/bot/documentation_link.py @@ -1,7 +1,20 @@ +from django.core.exceptions import ValidationError +from django.core.validators import RegexValidator from django.db import models from pydis_site.apps.api.models.mixins import ModelReprMixin +package_name_validator = RegexValidator( + regex=r"^[a-z0-9_]+$", + message="Package names can only consist of lowercase a-z letters, digits, and underscores." +) + + +def ends_with_slash_validator(string: str) -> None: + """Raise a ValidationError if `string` does not end with a slash.""" + if not string.endswith("/"): + raise ValidationError("The entered URL must end with a slash.") + class DocumentationLink(ModelReprMixin, models.Model): """A documentation link used by the `!docs` command of the bot.""" @@ -9,13 +22,15 @@ class DocumentationLink(ModelReprMixin, models.Model): package = models.CharField( primary_key=True, max_length=50, + validators=(package_name_validator,), help_text="The Python package name that this documentation link belongs to." ) base_url = models.URLField( help_text=( "The base URL from which documentation will be available for this project. " "Used to generate links to various symbols within this package." - ) + ), + validators=(ends_with_slash_validator,) ) inventory_url = models.URLField( help_text="The URL at which the Sphinx inventory is available for this package." diff --git a/pydis_site/apps/api/models/bot/infraction.py b/pydis_site/apps/api/models/bot/infraction.py index 7660cbba..60c1e8dd 100644 --- a/pydis_site/apps/api/models/bot/infraction.py +++ b/pydis_site/apps/api/models/bot/infraction.py @@ -15,7 +15,8 @@ class Infraction(ModelReprMixin, models.Model): ("mute", "Mute"), ("kick", "Kick"), ("ban", "Ban"), - ("superstar", "Superstar") + ("superstar", "Superstar"), + ("voice_ban", "Voice Ban"), ) inserted_at = models.DateTimeField( default=timezone.now, diff --git a/pydis_site/apps/api/models/bot/metricity.py b/pydis_site/apps/api/models/bot/metricity.py new file mode 100644 index 00000000..5daa5c66 --- /dev/null +++ b/pydis_site/apps/api/models/bot/metricity.py @@ -0,0 +1,132 @@ +from typing import List, Tuple + +from django.db import connections + +BLOCK_INTERVAL = 10 * 60 # 10 minute blocks + +EXCLUDE_CHANNELS = [ + "267659945086812160", # Bot commands + "607247579608121354" # SeasonalBot commands +] + + +class NotFound(Exception): + """Raised when an entity cannot be found.""" + + pass + + +class Metricity: + """Abstraction for a connection to the metricity database.""" + + def __init__(self): + self.cursor = connections['metricity'].cursor() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.cursor.close() + + def user(self, user_id: str) -> dict: + """Query a user's data.""" + # TODO: Swap this back to some sort of verified at date + columns = ["joined_at"] + query = f"SELECT {','.join(columns)} FROM users WHERE id = '%s'" + self.cursor.execute(query, [user_id]) + values = self.cursor.fetchone() + + if not values: + raise NotFound() + + return dict(zip(columns, values)) + + def total_messages(self, user_id: str) -> int: + """Query total number of messages for a user.""" + self.cursor.execute( + """ + SELECT + COUNT(*) + FROM messages + WHERE + author_id = '%s' + AND NOT is_deleted + AND NOT %s::varchar[] @> ARRAY[channel_id] + """, + [user_id, EXCLUDE_CHANNELS] + ) + values = self.cursor.fetchone() + + if not values: + raise NotFound() + + return values[0] + + def total_message_blocks(self, user_id: str) -> int: + """ + Query number of 10 minute blocks during which the user has been active. + + This metric prevents users from spamming to achieve the message total threshold. + """ + self.cursor.execute( + """ + SELECT + COUNT(*) + FROM ( + SELECT + (floor((extract('epoch' from created_at) / %s )) * %s) AS interval + FROM messages + WHERE + author_id='%s' + AND NOT is_deleted + AND NOT %s::varchar[] @> ARRAY[channel_id] + GROUP BY interval + ) block_query; + """, + [BLOCK_INTERVAL, BLOCK_INTERVAL, user_id, EXCLUDE_CHANNELS] + ) + values = self.cursor.fetchone() + + if not values: + raise NotFound() + + return values[0] + + def top_channel_activity(self, user_id: str) -> List[Tuple[str, int]]: + """ + Query the top three channels in which the user is most active. + + Help channels are grouped under "the help channels", + and off-topic channels are grouped under "off-topic". + """ + self.cursor.execute( + """ + SELECT + CASE + WHEN channels.name ILIKE 'help-%%' THEN 'the help channels' + WHEN channels.name ILIKE 'ot%%' THEN 'off-topic' + WHEN channels.name ILIKE '%%voice%%' THEN 'voice chats' + ELSE channels.name + END, + COUNT(1) + FROM + messages + LEFT JOIN channels ON channels.id = messages.channel_id + WHERE + author_id = '%s' AND NOT messages.is_deleted + GROUP BY + 1 + ORDER BY + 2 DESC + LIMIT + 3; + """, + [user_id] + ) + + values = self.cursor.fetchall() + + if not values: + raise NotFound() + + return values diff --git a/pydis_site/apps/api/models/bot/nomination.py b/pydis_site/apps/api/models/bot/nomination.py index 11b9e36e..221d8534 100644 --- a/pydis_site/apps/api/models/bot/nomination.py +++ b/pydis_site/apps/api/models/bot/nomination.py @@ -5,23 +5,12 @@ from pydis_site.apps.api.models.mixins import ModelReprMixin class Nomination(ModelReprMixin, models.Model): - """A helper nomination created by staff.""" + """A general helper nomination information created by staff.""" active = models.BooleanField( default=True, help_text="Whether this nomination is still relevant." ) - actor = models.ForeignKey( - User, - on_delete=models.CASCADE, - help_text="The staff member that nominated this user.", - related_name='nomination_set' - ) - reason = models.TextField( - help_text="Why this user was nominated.", - null=True, - blank=True - ) user = models.ForeignKey( User, on_delete=models.CASCADE, @@ -42,6 +31,10 @@ class Nomination(ModelReprMixin, models.Model): help_text="When the nomination was ended.", null=True ) + reviewed = models.BooleanField( + default=False, + help_text="Whether a review was made." + ) def __str__(self): """Representation that makes the target and state of the nomination immediately evident.""" @@ -52,3 +45,38 @@ class Nomination(ModelReprMixin, models.Model): """Set the ordering of nominations to most recent first.""" ordering = ("-inserted_at",) + + +class NominationEntry(ModelReprMixin, models.Model): + """A nomination entry created by a single staff member.""" + + nomination = models.ForeignKey( + Nomination, + on_delete=models.CASCADE, + help_text="The nomination this entry belongs to.", + related_name="entries" + ) + actor = models.ForeignKey( + User, + on_delete=models.CASCADE, + help_text="The staff member that nominated this user.", + related_name='nomination_set' + ) + reason = models.TextField( + help_text="Why the actor nominated this user.", + default="", + blank=True + ) + inserted_at = models.DateTimeField( + auto_now_add=True, + help_text="The creation date of this nomination entry." + ) + + class Meta: + """Meta options for NominationEntry model.""" + + verbose_name_plural = "nomination entries" + + # Set default ordering here to latest first + # so we don't need to define it everywhere + ordering = ("-inserted_at",) diff --git a/pydis_site/apps/api/models/log_entry.py b/pydis_site/apps/api/models/log_entry.py deleted file mode 100644 index 752cd2ca..00000000 --- a/pydis_site/apps/api/models/log_entry.py +++ /dev/null @@ -1,55 +0,0 @@ -from django.db import models -from django.utils import timezone - -from pydis_site.apps.api.models.mixins import ModelReprMixin - - -class LogEntry(ModelReprMixin, models.Model): - """A log entry generated by one of the PyDis applications.""" - - application = models.CharField( - max_length=20, - help_text="The application that generated this log entry.", - choices=( - ('bot', 'Bot'), - ('seasonalbot', 'Seasonalbot'), - ('site', 'Website') - ) - ) - logger_name = models.CharField( - max_length=100, - help_text="The name of the logger that generated this log entry." - ) - timestamp = models.DateTimeField( - default=timezone.now, - help_text="The date and time when this entry was created." - ) - level = models.CharField( - max_length=8, # 'critical' - choices=( - ('debug', 'Debug'), - ('info', 'Info'), - ('warning', 'Warning'), - ('error', 'Error'), - ('critical', 'Critical') - ), - help_text=( - "The logger level at which this entry was emitted. The levels " - "correspond to the Python `logging` levels." - ) - ) - module = models.CharField( - max_length=100, - help_text="The fully qualified path of the module generating this log line." - ) - line = models.PositiveSmallIntegerField( - help_text="The line at which the log line was emitted." - ) - message = models.TextField( - help_text="The textual content of the log line." - ) - - class Meta: - """Customizes the default generated plural name to valid English.""" - - verbose_name_plural = 'Log entries' diff --git a/pydis_site/apps/api/pagination.py b/pydis_site/apps/api/pagination.py new file mode 100644 index 00000000..2a325460 --- /dev/null +++ b/pydis_site/apps/api/pagination.py @@ -0,0 +1,49 @@ +import typing + +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.response import Response + + +class LimitOffsetPaginationExtended(LimitOffsetPagination): + """ + Extend LimitOffsetPagination to customise the default response. + + For example: + + ## Default response + >>> { + ... "count": 1, + ... "next": None, + ... "previous": None, + ... "results": [{ + ... "id": 6, + ... "inserted_at": "2021-01-26T21:13:35.477879Z", + ... "expires_at": None, + ... "active": False, + ... "user": 1, + ... "actor": 2, + ... "type": "warning", + ... "reason": null, + ... "hidden": false + ... }] + ... } + + ## Required response + >>> [{ + ... "id": 6, + ... "inserted_at": "2021-01-26T21:13:35.477879Z", + ... "expires_at": None, + ... "active": False, + ... "user": 1, + ... "actor": 2, + ... "type": "warning", + ... "reason": None, + ... "hidden": False + ... }] + """ + + default_limit = 100 + + def get_paginated_response(self, data: typing.Any) -> Response: + """Override to skip metadata i.e. `count`, `next`, and `previous`.""" + return Response(data) diff --git a/pydis_site/apps/api/serializers.py b/pydis_site/apps/api/serializers.py index 90bd6f91..f47bedca 100644 --- a/pydis_site/apps/api/serializers.py +++ b/pydis_site/apps/api/serializers.py @@ -1,7 +1,16 @@ """Converters from Django models to data interchange formats and back.""" -from rest_framework.serializers import ModelSerializer, PrimaryKeyRelatedField, ValidationError +from django.db.models.query import QuerySet +from django.db.utils import IntegrityError +from rest_framework.exceptions import NotFound +from rest_framework.serializers import ( + IntegerField, + ListSerializer, + ModelSerializer, + PrimaryKeyRelatedField, + ValidationError +) +from rest_framework.settings import api_settings from rest_framework.validators import UniqueTogetherValidator -from rest_framework_bulk import BulkSerializerMixin from .models import ( BotSetting, @@ -9,9 +18,9 @@ from .models import ( DocumentationLink, FilterList, Infraction, - LogEntry, MessageDeletionContext, Nomination, + NominationEntry, OffTopicChannelName, OffensiveMessage, Reminder, @@ -159,7 +168,7 @@ class InfractionSerializer(ModelSerializer): raise ValidationError({'expires_at': [f'{infr_type} infractions cannot expire.']}) hidden = attrs.get('hidden') - if hidden and infr_type in ('superstar', 'warning'): + if hidden and infr_type in ('superstar', 'warning', 'voice_ban'): raise ValidationError({'hidden': [f'{infr_type} infractions cannot be hidden.']}) if not hidden and infr_type in ('note', ): @@ -191,19 +200,6 @@ class ExpandedInfractionSerializer(InfractionSerializer): return ret -class LogEntrySerializer(ModelSerializer): - """A class providing (de-)serialization of `LogEntry` instances.""" - - class Meta: - """Metadata defined for the Django REST Framework.""" - - model = LogEntry - fields = ( - 'application', 'logger_name', 'timestamp', - 'level', 'module', 'line', 'message' - ) - - class OffTopicChannelNameSerializer(ModelSerializer): """A class providing (de-)serialization of `OffTopicChannelName` instances.""" @@ -249,27 +245,130 @@ class RoleSerializer(ModelSerializer): fields = ('id', 'name', 'colour', 'permissions', 'position') -class UserSerializer(BulkSerializerMixin, ModelSerializer): +class UserListSerializer(ListSerializer): + """List serializer for User model to handle bulk updates.""" + + def create(self, validated_data: list) -> list: + """Override create method to optimize django queries.""" + new_users = [] + seen = set() + + for user_dict in validated_data: + if user_dict["id"] in seen: + raise ValidationError( + {"id": [f"User with ID {user_dict['id']} given multiple times."]} + ) + seen.add(user_dict["id"]) + new_users.append(User(**user_dict)) + + User.objects.bulk_create(new_users, ignore_conflicts=True) + return [] + + def update(self, queryset: QuerySet, validated_data: list) -> list: + """ + Override update method to support bulk updates. + + ref:https://www.django-rest-framework.org/api-guide/serializers/#customizing-multiple-update + """ + object_ids = set() + + for data in validated_data: + try: + if data["id"] in object_ids: + # If request data contains users with same ID. + raise ValidationError( + {"id": [f"User with ID {data['id']} given multiple times."]} + ) + except KeyError: + # If user ID not provided in request body. + raise ValidationError( + {"id": ["This field is required."]} + ) + object_ids.add(data["id"]) + + # filter queryset + filtered_instances = queryset.filter(id__in=object_ids) + + instance_mapping = {user.id: user for user in filtered_instances} + + updated = [] + fields_to_update = set() + for user_data in validated_data: + for key in user_data: + fields_to_update.add(key) + + try: + user = instance_mapping[user_data["id"]] + except KeyError: + raise NotFound({"detail": f"User with id {user_data['id']} not found."}) + + user.__dict__.update(user_data) + updated.append(user) + + fields_to_update.remove("id") + + if not fields_to_update: + # Raise ValidationError when only id field is given. + raise ValidationError( + {api_settings.NON_FIELD_ERRORS_KEY: ["Insufficient data provided."]} + ) + + User.objects.bulk_update(updated, fields_to_update) + return updated + + +class UserSerializer(ModelSerializer): """A class providing (de-)serialization of `User` instances.""" + # ID field must be explicitly set as the default id field is read-only. + id = IntegerField(min_value=0) + class Meta: """Metadata defined for the Django REST Framework.""" model = User fields = ('id', 'name', 'discriminator', 'roles', 'in_guild') depth = 1 + list_serializer_class = UserListSerializer + + def create(self, validated_data: dict) -> User: + """Override create method to catch IntegrityError.""" + try: + return super().create(validated_data) + except IntegrityError: + raise ValidationError({"id": ["User with ID already present."]}) + + +class NominationEntrySerializer(ModelSerializer): + """A class providing (de-)serialization of `NominationEntry` instances.""" + + # We need to define it here, because we don't want that nomination ID + # return inside nomination response entry, because ID is already available + # as top-level field. Queryset is required if field is not read only. + nomination = PrimaryKeyRelatedField( + queryset=Nomination.objects.all(), + write_only=True + ) + + class Meta: + """Metadata defined for the Django REST framework.""" + + model = NominationEntry + fields = ('nomination', 'actor', 'reason', 'inserted_at') class NominationSerializer(ModelSerializer): """A class providing (de-)serialization of `Nomination` instances.""" + entries = NominationEntrySerializer(many=True, read_only=True) + class Meta: """Metadata defined for the Django REST Framework.""" model = Nomination fields = ( - 'id', 'active', 'actor', 'reason', 'user', - 'inserted_at', 'end_reason', 'ended_at') + 'id', 'active', 'user', 'inserted_at', 'end_reason', 'ended_at', 'reviewed', 'entries' + ) class OffensiveMessageSerializer(ModelSerializer): diff --git a/pydis_site/apps/api/tests/test_dblogger.py b/pydis_site/apps/api/tests/test_dblogger.py deleted file mode 100644 index bb19f297..00000000 --- a/pydis_site/apps/api/tests/test_dblogger.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -from datetime import datetime - -from django.test import TestCase - -from ..dblogger import DatabaseLogHandler -from ..models import LogEntry - - -class DatabaseLogHandlerTests(TestCase): - def test_logs_to_database(self): - module_basename = __name__.split('.')[-1] - logger = logging.getLogger(__name__) - logger.handlers = [DatabaseLogHandler()] - logger.warning("I am a test case!") - - # Ensure we only have a single record in the database - # after the logging call above. - [entry] = LogEntry.objects.all() - - self.assertEqual(entry.application, 'site') - self.assertEqual(entry.logger_name, __name__) - self.assertIsInstance(entry.timestamp, datetime) - self.assertEqual(entry.level, 'warning') - self.assertEqual(entry.module, module_basename) - self.assertIsInstance(entry.line, int) - self.assertEqual(entry.message, "I am a test case!") diff --git a/pydis_site/apps/api/tests/test_documentation_links.py b/pydis_site/apps/api/tests/test_documentation_links.py index e560a2fd..39fb08f3 100644 --- a/pydis_site/apps/api/tests/test_documentation_links.py +++ b/pydis_site/apps/api/tests/test_documentation_links.py @@ -60,7 +60,7 @@ class DetailLookupDocumentationLinkAPITests(APISubdomainTestCase): def setUpTestData(cls): cls.doc_link = DocumentationLink.objects.create( package='testpackage', - base_url='https://example.com', + base_url='https://example.com/', inventory_url='https://example.com' ) @@ -108,6 +108,17 @@ class DetailLookupDocumentationLinkAPITests(APISubdomainTestCase): self.assertEqual(response.status_code, 400) + def test_create_invalid_package_name_returns_400(self): + test_cases = ("InvalidPackage", "invalid package", "i\u0150valid") + for case in test_cases: + with self.subTest(package_name=case): + body = self.doc_json.copy() + body['package'] = case + url = reverse('bot:documentationlink-list', host='api') + response = self.client.post(url, data=body) + + self.assertEqual(response.status_code, 400) + class DocumentationLinkCreationTests(APISubdomainTestCase): def setUp(self): @@ -115,7 +126,7 @@ class DocumentationLinkCreationTests(APISubdomainTestCase): self.body = { 'package': 'example', - 'base_url': 'https://example.com', + 'base_url': 'https://example.com/', 'inventory_url': 'https://docs.example.com' } diff --git a/pydis_site/apps/api/tests/test_infractions.py b/pydis_site/apps/api/tests/test_infractions.py index 93ef8171..82b497aa 100644 --- a/pydis_site/apps/api/tests/test_infractions.py +++ b/pydis_site/apps/api/tests/test_infractions.py @@ -512,6 +512,36 @@ class CreationTests(APISubdomainTestCase): ) +class InfractionDeletionTests(APISubdomainTestCase): + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create( + id=9876, + name='Unknown user', + discriminator=9876, + ) + + cls.warning = Infraction.objects.create( + user_id=cls.user.id, + actor_id=cls.user.id, + type='warning', + active=False + ) + + def test_delete_unknown_infraction_returns_404(self): + url = reverse('bot:infraction-detail', args=('something',), host='api') + response = self.client.delete(url) + + self.assertEqual(response.status_code, 404) + + def test_delete_known_infraction_returns_204(self): + url = reverse('bot:infraction-detail', args=(self.warning.id,), host='api') + response = self.client.delete(url) + + self.assertEqual(response.status_code, 204) + self.assertRaises(Infraction.DoesNotExist, Infraction.objects.get, id=self.warning.id) + + class ExpandedTests(APISubdomainTestCase): @classmethod def setUpTestData(cls): diff --git a/pydis_site/apps/api/tests/test_models.py b/pydis_site/apps/api/tests/test_models.py index 853e6621..66052e01 100644 --- a/pydis_site/apps/api/tests/test_models.py +++ b/pydis_site/apps/api/tests/test_models.py @@ -10,6 +10,7 @@ from pydis_site.apps.api.models import ( Message, MessageDeletionContext, Nomination, + NominationEntry, OffTopicChannelName, OffensiveMessage, Reminder, @@ -37,17 +38,11 @@ class StringDunderMethodTests(SimpleTestCase): def setUp(self): self.nomination = Nomination( id=123, - actor=User( - id=9876, - name='Mr. Hemlock', - discriminator=6666, - ), user=User( id=9876, name="Hemlock's Cat", discriminator=7777, ), - reason="He purrrrs like the best!", ) self.objects = ( @@ -135,6 +130,15 @@ class StringDunderMethodTests(SimpleTestCase): ), content="oh no", expiration=dt(5018, 11, 20, 15, 52, tzinfo=timezone.utc) + ), + NominationEntry( + nomination_id=self.nomination.id, + actor=User( + id=9876, + name='Mr. Hemlock', + discriminator=6666, + ), + reason="He purrrrs like the best!", ) ) diff --git a/pydis_site/apps/api/tests/test_nominations.py b/pydis_site/apps/api/tests/test_nominations.py index b37135f8..9cefbd8f 100644 --- a/pydis_site/apps/api/tests/test_nominations.py +++ b/pydis_site/apps/api/tests/test_nominations.py @@ -3,7 +3,7 @@ from datetime import datetime as dt, timedelta, timezone from django_hosts.resolvers import reverse from .base import APISubdomainTestCase -from ..models import Nomination, User +from ..models import Nomination, NominationEntry, User class CreationTests(APISubdomainTestCase): @@ -14,6 +14,11 @@ class CreationTests(APISubdomainTestCase): name='joe dart', discriminator=1111, ) + cls.user2 = User.objects.create( + id=9876, + name='Who?', + discriminator=1234 + ) def test_accepts_valid_data(self): url = reverse('bot:nomination-list', host='api') @@ -27,17 +32,39 @@ class CreationTests(APISubdomainTestCase): self.assertEqual(response.status_code, 201) nomination = Nomination.objects.get(id=response.json()['id']) + nomination_entry = NominationEntry.objects.get( + nomination_id=nomination.id, + actor_id=self.user.id + ) self.assertAlmostEqual( nomination.inserted_at, dt.now(timezone.utc), delta=timedelta(seconds=2) ) self.assertEqual(nomination.user.id, data['user']) - self.assertEqual(nomination.actor.id, data['actor']) - self.assertEqual(nomination.reason, data['reason']) + self.assertEqual(nomination_entry.reason, data['reason']) self.assertEqual(nomination.active, True) - def test_returns_400_on_second_active_nomination(self): + def test_returns_200_on_second_active_nomination_by_different_user(self): + url = reverse('bot:nomination-list', host='api') + first_data = { + 'actor': self.user.id, + 'reason': 'Joe Dart on Fender Bass', + 'user': self.user.id, + } + second_data = { + 'actor': self.user2.id, + 'reason': 'Great user', + 'user': self.user.id + } + + response1 = self.client.post(url, data=first_data) + self.assertEqual(response1.status_code, 201) + + response2 = self.client.post(url, data=second_data) + self.assertEqual(response2.status_code, 201) + + def test_returns_400_on_second_active_nomination_by_existing_nominator(self): url = reverse('bot:nomination-list', host='api') data = { 'actor': self.user.id, @@ -51,7 +78,7 @@ class CreationTests(APISubdomainTestCase): response2 = self.client.post(url, data=data) self.assertEqual(response2.status_code, 400) self.assertEqual(response2.json(), { - 'active': ['There can only be one active nomination.'] + 'actor': ['This actor has already endorsed this nomination.'] }) def test_returns_400_for_missing_user(self): @@ -189,30 +216,40 @@ class NominationTests(APISubdomainTestCase): ) cls.active_nomination = Nomination.objects.create( - user=cls.user, + user=cls.user + ) + cls.active_nomination_entry = NominationEntry.objects.create( + nomination=cls.active_nomination, actor=cls.user, reason="He's pretty funky" ) cls.inactive_nomination = Nomination.objects.create( user=cls.user, - actor=cls.user, - reason="He's pretty funky", active=False, end_reason="His neck couldn't hold the funk", ended_at="5018-11-20T15:52:00+00:00" ) + cls.inactive_nomination_entry = NominationEntry.objects.create( + nomination=cls.inactive_nomination, + actor=cls.user, + reason="He's pretty funky" + ) - def test_returns_200_update_reason_on_active(self): + def test_returns_200_update_reason_on_active_with_actor(self): url = reverse('bot:nomination-detail', args=(self.active_nomination.id,), host='api') data = { - 'reason': "He's one funky duck" + 'reason': "He's one funky duck", + 'actor': self.user.id } response = self.client.patch(url, data=data) self.assertEqual(response.status_code, 200) - nomination = Nomination.objects.get(id=response.json()['id']) - self.assertEqual(nomination.reason, data['reason']) + nomination_entry = NominationEntry.objects.get( + nomination_id=response.json()['id'], + actor_id=self.user.id + ) + self.assertEqual(nomination_entry.reason, data['reason']) def test_returns_400_on_frozen_field_update(self): url = reverse('bot:nomination-detail', args=(self.active_nomination.id,), host='api') @@ -241,14 +278,18 @@ class NominationTests(APISubdomainTestCase): def test_returns_200_update_reason_on_inactive(self): url = reverse('bot:nomination-detail', args=(self.inactive_nomination.id,), host='api') data = { - 'reason': "He's one funky duck" + 'reason': "He's one funky duck", + 'actor': self.user.id } response = self.client.patch(url, data=data) self.assertEqual(response.status_code, 200) - nomination = Nomination.objects.get(id=response.json()['id']) - self.assertEqual(nomination.reason, data['reason']) + nomination_entry = NominationEntry.objects.get( + nomination_id=response.json()['id'], + actor_id=self.user.id + ) + self.assertEqual(nomination_entry.reason, data['reason']) def test_returns_200_update_end_reason_on_inactive(self): url = reverse('bot:nomination-detail', args=(self.inactive_nomination.id,), host='api') @@ -442,3 +483,50 @@ class NominationTests(APISubdomainTestCase): infractions = response.json() self.assertEqual(len(infractions), 2) + + def test_patch_nomination_set_reviewed_of_active_nomination(self): + url = reverse('api:nomination-detail', args=(self.active_nomination.id,), host='api') + data = {'reviewed': True} + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 200) + + def test_patch_nomination_set_reviewed_of_inactive_nomination(self): + url = reverse('api:nomination-detail', args=(self.inactive_nomination.id,), host='api') + data = {'reviewed': True} + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'reviewed': ['This field cannot be set if the nomination is inactive.'] + }) + + def test_patch_nomination_set_reviewed_and_end(self): + url = reverse('api:nomination-detail', args=(self.active_nomination.id,), host='api') + data = {'reviewed': True, 'active': False, 'end_reason': "What?"} + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'reviewed': ['This field cannot be set while you are ending a nomination.'] + }) + + def test_modifying_reason_without_actor(self): + url = reverse('api:nomination-detail', args=(self.active_nomination.id,), host='api') + data = {'reason': 'That is my reason!'} + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'actor': ['This field is required when editing the reason.'] + }) + + def test_modifying_reason_with_unknown_actor(self): + url = reverse('api:nomination-detail', args=(self.active_nomination.id,), host='api') + data = {'reason': 'That is my reason!', 'actor': 90909090909090} + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), { + 'actor': ["The actor doesn't exist or has not nominated the user."] + }) diff --git a/pydis_site/apps/api/tests/test_users.py b/pydis_site/apps/api/tests/test_users.py index a02fce8a..c43b916a 100644 --- a/pydis_site/apps/api/tests/test_users.py +++ b/pydis_site/apps/api/tests/test_users.py @@ -1,7 +1,11 @@ +from unittest.mock import patch + +from django.core.exceptions import ObjectDoesNotExist from django_hosts.resolvers import reverse from .base import APISubdomainTestCase from ..models import Role, User +from ..models.bot.metricity import NotFound class UnauthedUserAPITests(APISubdomainTestCase): @@ -45,6 +49,13 @@ class CreationTests(APISubdomainTestCase): position=1 ) + cls.user = User.objects.create( + id=11, + name="Name doesn't matter.", + discriminator=1122, + in_guild=True + ) + def test_accepts_valid_data(self): url = reverse('bot:user-list', host='api') data = { @@ -89,7 +100,7 @@ class CreationTests(APISubdomainTestCase): response = self.client.post(url, data=data) self.assertEqual(response.status_code, 201) - self.assertEqual(response.json(), data) + self.assertEqual(response.json(), []) def test_returns_400_for_unknown_role_id(self): url = reverse('bot:user-list', host='api') @@ -115,6 +126,176 @@ class CreationTests(APISubdomainTestCase): response = self.client.post(url, data=data) self.assertEqual(response.status_code, 400) + def test_returns_400_for_user_recreation(self): + """Return 201 if User is already present in database as it skips User creation.""" + url = reverse('bot:user-list', host='api') + data = [{ + 'id': 11, + 'name': 'You saw nothing.', + 'discriminator': 112, + 'in_guild': True + }] + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 201) + + def test_returns_400_for_duplicate_request_users(self): + """Return 400 if 2 Users with same ID is passed in the request data.""" + url = reverse('bot:user-list', host='api') + data = [ + { + 'id': 11, + 'name': 'You saw nothing.', + 'discriminator': 112, + 'in_guild': True + }, + { + 'id': 11, + 'name': 'You saw nothing part 2.', + 'discriminator': 1122, + 'in_guild': False + } + ] + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 400) + + def test_returns_400_for_existing_user(self): + """Returns 400 if user is already present in DB.""" + url = reverse('bot:user-list', host='api') + data = { + 'id': 11, + 'name': 'You saw nothing part 3.', + 'discriminator': 1122, + 'in_guild': True + } + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 400) + + +class MultiPatchTests(APISubdomainTestCase): + @classmethod + def setUpTestData(cls): + cls.role_developer = Role.objects.create( + id=159, + name="Developer", + colour=2, + permissions=0b01010010101, + position=10, + ) + cls.user_1 = User.objects.create( + id=1, + name="Patch test user 1.", + discriminator=1111, + in_guild=True + ) + cls.user_2 = User.objects.create( + id=2, + name="Patch test user 2.", + discriminator=2222, + in_guild=True + ) + + def test_multiple_users_patch(self): + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + "id": 1, + "name": "User 1 patched!", + "discriminator": 1010, + "roles": [self.role_developer.id], + "in_guild": False + }, + { + "id": 2, + "name": "User 2 patched!" + } + ] + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0], data[0]) + + user_2 = User.objects.get(id=2) + self.assertEqual(user_2.name, data[1]["name"]) + + def test_returns_400_for_missing_user_id(self): + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + "name": "I am ghost user!", + "discriminator": 1010, + "roles": [self.role_developer.id], + "in_guild": False + }, + { + "name": "patch me? whats my id?" + } + ] + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + + def test_returns_404_for_not_found_user(self): + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + "id": 1, + "name": "User 1 patched again!!!", + "discriminator": 1010, + "roles": [self.role_developer.id], + "in_guild": False + }, + { + "id": 22503405, + "name": "User unknown not patched!" + } + ] + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 404) + + def test_returns_400_for_bad_data(self): + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + "id": 1, + "in_guild": "Catch me!" + }, + { + "id": 2, + "discriminator": "find me!" + } + ] + + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + + def test_returns_400_for_insufficient_data(self): + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + "id": 1, + }, + { + "id": 2, + } + ] + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + + def test_returns_400_for_duplicate_request_users(self): + """Return 400 if 2 Users with same ID is passed in the request data.""" + url = reverse("bot:user-bulk-patch", host="api") + data = [ + { + 'id': 1, + 'name': 'You saw nothing.', + }, + { + 'id': 1, + 'name': 'You saw nothing part 2.', + } + ] + response = self.client.patch(url, data=data) + self.assertEqual(response.status_code, 400) + class UserModelTests(APISubdomainTestCase): @classmethod @@ -170,3 +351,157 @@ class UserModelTests(APISubdomainTestCase): def test_correct_username_formatting(self): """Tests the username property with both name and discriminator formatted together.""" self.assertEqual(self.user_with_roles.username, "Test User with two roles#0001") + + +class UserPaginatorTests(APISubdomainTestCase): + @classmethod + def setUpTestData(cls): + users = [] + for i in range(1, 10_001): + users.append(User( + id=i, + name=f"user{i}", + discriminator=1111, + in_guild=True + )) + cls.users = User.objects.bulk_create(users) + + def test_returns_single_page_response(self): + url = reverse("bot:user-list", host="api") + response = self.client.get(url).json() + self.assertIsNone(response["next_page_no"]) + self.assertIsNone(response["previous_page_no"]) + + def test_returns_next_page_number(self): + User.objects.create( + id=10_001, + name="user10001", + discriminator=1111, + in_guild=True + ) + url = reverse("bot:user-list", host="api") + response = self.client.get(url).json() + self.assertEqual(2, response["next_page_no"]) + + def test_returns_previous_page_number(self): + User.objects.create( + id=10_001, + name="user10001", + discriminator=1111, + in_guild=True + ) + url = reverse("bot:user-list", host="api") + response = self.client.get(url, {"page": 2}).json() + self.assertEqual(1, response["previous_page_no"]) + + +class UserMetricityTests(APISubdomainTestCase): + @classmethod + def setUpTestData(cls): + User.objects.create( + id=0, + name="Test user", + discriminator=1, + in_guild=True, + ) + + def test_get_metricity_data(self): + # Given + joined_at = "foo" + total_messages = 1 + total_blocks = 1 + self.mock_metricity_user(joined_at, total_messages, total_blocks, []) + + # When + url = reverse('bot:user-metricity-data', args=[0], host='api') + response = self.client.get(url) + + # Then + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), { + "joined_at": joined_at, + "total_messages": total_messages, + "voice_banned": False, + "activity_blocks": total_blocks + }) + + def test_no_metricity_user(self): + # Given + self.mock_no_metricity_user() + + # When + url = reverse('bot:user-metricity-data', args=[0], host='api') + response = self.client.get(url) + + # Then + self.assertEqual(response.status_code, 404) + + def test_no_metricity_user_for_review(self): + # Given + self.mock_no_metricity_user() + + # When + url = reverse('bot:user-metricity-review-data', args=[0], host='api') + response = self.client.get(url) + + # Then + self.assertEqual(response.status_code, 404) + + def test_metricity_voice_banned(self): + cases = [ + {'exception': None, 'voice_banned': True}, + {'exception': ObjectDoesNotExist, 'voice_banned': False}, + ] + + self.mock_metricity_user("foo", 1, 1, [["bar", 1]]) + + for case in cases: + with self.subTest(exception=case['exception'], voice_banned=case['voice_banned']): + with patch("pydis_site.apps.api.viewsets.bot.user.Infraction.objects.get") as p: + p.side_effect = case['exception'] + + url = reverse('bot:user-metricity-data', args=[0], host='api') + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["voice_banned"], case["voice_banned"]) + + def test_metricity_review_data(self): + # Given + joined_at = "foo" + total_messages = 10 + total_blocks = 1 + channel_activity = [["bar", 4], ["buzz", 6]] + self.mock_metricity_user(joined_at, total_messages, total_blocks, channel_activity) + + # When + url = reverse('bot:user-metricity-review-data', args=[0], host='api') + response = self.client.get(url) + + # Then + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), { + "joined_at": joined_at, + "top_channel_activity": channel_activity, + "total_messages": total_messages + }) + + def mock_metricity_user(self, joined_at, total_messages, total_blocks, top_channel_activity): + patcher = patch("pydis_site.apps.api.viewsets.bot.user.Metricity") + self.metricity = patcher.start() + self.addCleanup(patcher.stop) + self.metricity = self.metricity.return_value.__enter__.return_value + self.metricity.user.return_value = dict(joined_at=joined_at) + self.metricity.total_messages.return_value = total_messages + self.metricity.total_message_blocks.return_value = total_blocks + self.metricity.top_channel_activity.return_value = top_channel_activity + + def mock_no_metricity_user(self): + patcher = patch("pydis_site.apps.api.viewsets.bot.user.Metricity") + self.metricity = patcher.start() + self.addCleanup(patcher.stop) + self.metricity = self.metricity.return_value.__enter__.return_value + self.metricity.user.side_effect = NotFound() + self.metricity.total_messages.side_effect = NotFound() + self.metricity.total_message_blocks.side_effect = NotFound() + self.metricity.top_channel_activity.side_effect = NotFound() diff --git a/pydis_site/apps/api/urls.py b/pydis_site/apps/api/urls.py index 4dbf93db..2e1ef0b4 100644 --- a/pydis_site/apps/api/urls.py +++ b/pydis_site/apps/api/urls.py @@ -8,7 +8,6 @@ from .viewsets import ( DocumentationLinkViewSet, FilterListViewSet, InfractionViewSet, - LogEntryViewSet, NominationViewSet, OffTopicChannelNameViewSet, OffensiveMessageViewSet, @@ -71,7 +70,6 @@ urlpatterns = ( # # from django_hosts.resolvers import reverse path('bot/', include((bot_router.urls, 'api'), namespace='bot')), - path('logs', LogEntryViewSet.as_view({'post': 'create'}), name='logs'), path('healthcheck', HealthcheckView.as_view(), name='healthcheck'), path('rules', RulesView.as_view(), name='rules') ) diff --git a/pydis_site/apps/api/views.py b/pydis_site/apps/api/views.py index 7ac56641..0d126051 100644 --- a/pydis_site/apps/api/views.py +++ b/pydis_site/apps/api/views.py @@ -135,8 +135,9 @@ class RulesView(APIView): ), ( "Do not provide or request help on projects that may break laws, " - "breach terms of services, be considered malicious/inappropriate " - "or be for graded coursework/exams." + "breach terms of services, be considered malicious or inappropriate. " + "Do not help with ongoing exams. Do not provide or request solutions " + "for graded assignments, although general guidance is okay." ), ( "No spamming or unapproved advertising, including requests for paid work. " diff --git a/pydis_site/apps/api/viewsets/__init__.py b/pydis_site/apps/api/viewsets/__init__.py index dfbb880d..f133e77f 100644 --- a/pydis_site/apps/api/viewsets/__init__.py +++ b/pydis_site/apps/api/viewsets/__init__.py @@ -12,4 +12,3 @@ from .bot import ( RoleViewSet, UserViewSet ) -from .log_entry import LogEntryViewSet diff --git a/pydis_site/apps/api/viewsets/bot/infraction.py b/pydis_site/apps/api/viewsets/bot/infraction.py index edec0a1e..bd512ddd 100644 --- a/pydis_site/apps/api/viewsets/bot/infraction.py +++ b/pydis_site/apps/api/viewsets/bot/infraction.py @@ -5,6 +5,7 @@ from rest_framework.exceptions import ValidationError from rest_framework.filters import OrderingFilter, SearchFilter from rest_framework.mixins import ( CreateModelMixin, + DestroyModelMixin, ListModelMixin, RetrieveModelMixin ) @@ -12,13 +13,20 @@ from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet from pydis_site.apps.api.models.bot.infraction import Infraction +from pydis_site.apps.api.pagination import LimitOffsetPaginationExtended from pydis_site.apps.api.serializers import ( ExpandedInfractionSerializer, InfractionSerializer ) -class InfractionViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, GenericViewSet): +class InfractionViewSet( + CreateModelMixin, + RetrieveModelMixin, + ListModelMixin, + GenericViewSet, + DestroyModelMixin +): """ View providing CRUD operations on infractions for Discord users. @@ -31,6 +39,8 @@ class InfractionViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge - **active** `bool`: whether the infraction is still active - **actor__id** `int`: snowflake of the user which applied the infraction - **hidden** `bool`: whether the infraction is a shadow infraction + - **limit** `int`: number of results return per page (default 100) + - **offset** `int`: the initial index from which to return the results (default 0) - **search** `str`: regular expression applied to the infraction's reason - **type** `str`: the type of the infraction - **user__id** `int`: snowflake of the user to which the infraction was applied @@ -39,6 +49,7 @@ class InfractionViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge Invalid query parameters are ignored. #### Response format + Response is paginated but the result is returned without any pagination metadata. >>> [ ... { ... 'id': 5, @@ -108,6 +119,13 @@ class InfractionViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge - 400: if a field in the request body is invalid or disallowed - 404: if an infraction with the given `id` could not be found + ### DELETE /bot/infractions/<id:int> + Delete the infraction with the given `id`. + + #### Status codes + - 204: returned on success + - 404: if a infraction with the given `id` does not exist + ### Expanded routes All routes support expansion of `user` and `actor` in responses. To use an expanded route, append `/expanded` to the end of the route e.g. `GET /bot/infractions/expanded`. @@ -119,6 +137,7 @@ class InfractionViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge serializer_class = InfractionSerializer queryset = Infraction.objects.all() + pagination_class = LimitOffsetPaginationExtended filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter) filter_fields = ('user__id', 'actor__id', 'active', 'hidden', 'type') search_fields = ('$reason',) diff --git a/pydis_site/apps/api/viewsets/bot/nomination.py b/pydis_site/apps/api/viewsets/bot/nomination.py index cf6e262f..144daab0 100644 --- a/pydis_site/apps/api/viewsets/bot/nomination.py +++ b/pydis_site/apps/api/viewsets/bot/nomination.py @@ -14,8 +14,8 @@ from rest_framework.mixins import ( from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet -from pydis_site.apps.api.models.bot import Nomination -from pydis_site.apps.api.serializers import NominationSerializer +from pydis_site.apps.api.models.bot import Nomination, NominationEntry +from pydis_site.apps.api.serializers import NominationEntrySerializer, NominationSerializer class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, GenericViewSet): @@ -29,7 +29,6 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge #### Query parameters - **active** `bool`: whether the nomination is still active - - **actor__id** `int`: snowflake of the user who nominated the user - **user__id** `int`: snowflake of the user who received the nomination - **ordering** `str`: comma-separated sequence of fields to order the returned results @@ -40,12 +39,18 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge ... { ... 'id': 1, ... 'active': false, - ... 'actor': 336843820513755157, - ... 'reason': 'They know how to explain difficult concepts', ... 'user': 336843820513755157, ... 'inserted_at': '2019-04-25T14:02:37.775587Z', ... 'end_reason': 'They were helpered after a staff-vote', - ... 'ended_at': '2019-04-26T15:12:22.123587Z' + ... 'ended_at': '2019-04-26T15:12:22.123587Z', + ... 'entries': [ + ... { + ... 'actor': 336843820513755157, + ... 'reason': 'They know how to explain difficult concepts', + ... 'inserted_at': '2019-04-25T14:02:37.775587Z' + ... } + ... ], + ... 'reviewed': true ... } ... ] @@ -59,12 +64,18 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge >>> { ... 'id': 1, ... 'active': true, - ... 'actor': 336843820513755157, - ... 'reason': 'They know how to explain difficult concepts', ... 'user': 336843820513755157, ... 'inserted_at': '2019-04-25T14:02:37.775587Z', ... 'end_reason': 'They were helpered after a staff-vote', - ... 'ended_at': '2019-04-26T15:12:22.123587Z' + ... 'ended_at': '2019-04-26T15:12:22.123587Z', + ... 'entries': [ + ... { + ... 'actor': 336843820513755157, + ... 'reason': 'They know how to explain difficult concepts', + ... 'inserted_at': '2019-04-25T14:02:37.775587Z' + ... } + ... ], + ... 'reviewed': false ... } ### Status codes @@ -75,8 +86,9 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge Create a new, active nomination returns the created nominations. The `user`, `reason` and `actor` fields are required and the `user` and `actor` need to know by the site. Providing other valid fields - is not allowed and invalid fields are ignored. A `user` is only - allowed one active nomination at a time. + is not allowed and invalid fields are ignored. If `user` already has an + active nomination, a new nomination entry will be created and assigned to the + active nomination. #### Request body >>> { @@ -91,7 +103,6 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge #### Status codes - 201: returned on success - 400: returned on failure for one of the following reasons: - - A user already has an active nomination; - The `user` or `actor` are unknown to the site; - The request contained a field that cannot be set at creation. @@ -102,16 +113,18 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge 1. Updating the `reason` of `active` nomination; 2. Ending an `active` nomination; 3. Updating the `end_reason` or `reason` field of an `inactive` nomination. + 4. Updating `reviewed` field of `active` nomination. While the response format and status codes are the same for all three operations (see below), the request bodies vary depending on the operation. For all operations it holds that providing other valid fields is not allowed and invalid fields are ignored. - ### 1. Updating the `reason` of `active` nomination + ### 1. Updating the `reason` of `active` nomination. The `actor` field is required. #### Request body >>> { ... 'reason': 'He would make a great helper', + ... 'actor': 409107086526644234 ... } #### Response format @@ -133,24 +146,35 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge See operation 1 for the response format and status codes. ### 3. Updating the `end_reason` or `reason` field of an `inactive` nomination. + Actor field is required when updating reason. #### Request body >>> { ... 'reason': 'Updated reason for this nomination', + ... 'actor': 409107086526644234, ... 'end_reason': 'Updated end_reason for this nomination', ... } Note: The request body may contain either or both fields. See operation 1 for the response format and status codes. + + ### 4. Setting nomination `reviewed` + + #### Request body + >>> { + ... 'reviewed': True + ... } + + See operation 1 for the response format and status codes. """ serializer_class = NominationSerializer queryset = Nomination.objects.all() filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter) - filter_fields = ('user__id', 'actor__id', 'active') - frozen_fields = ('id', 'actor', 'inserted_at', 'user', 'ended_at') - frozen_on_create = ('ended_at', 'end_reason', 'active', 'inserted_at') + filter_fields = ('user__id', 'active') + frozen_fields = ('id', 'inserted_at', 'user', 'ended_at') + frozen_on_create = ('ended_at', 'end_reason', 'active', 'inserted_at', 'reviewed') def create(self, request: HttpRequest, *args, **kwargs) -> Response: """ @@ -163,19 +187,50 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge raise ValidationError({field: ['This field cannot be set at creation.']}) user_id = request.data.get("user") - if Nomination.objects.filter(active=True, user__id=user_id).exists(): - raise ValidationError({'active': ['There can only be one active nomination.']}) + nomination_filter = Nomination.objects.filter(active=True, user__id=user_id) + + if not nomination_filter.exists(): + serializer = NominationSerializer( + data=ChainMap( + request.data, + {"active": True} + ) + ) + serializer.is_valid(raise_exception=True) + nomination = Nomination.objects.create(**serializer.validated_data) - serializer = self.get_serializer( - data=ChainMap( - request.data, - {"active": True} + # The serializer will truncate and get rid of excessive data + entry_serializer = NominationEntrySerializer( + data=ChainMap(request.data, {"nomination": nomination.id}) ) + entry_serializer.is_valid(raise_exception=True) + NominationEntry.objects.create(**entry_serializer.validated_data) + + data = NominationSerializer(nomination).data + + headers = self.get_success_headers(data) + return Response(data, status=status.HTTP_201_CREATED, headers=headers) + + entry_serializer = NominationEntrySerializer( + data=ChainMap(request.data, {"nomination": nomination_filter[0].id}) ) - serializer.is_valid(raise_exception=True) - self.perform_create(serializer) - headers = self.get_success_headers(serializer.data) - return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + entry_serializer.is_valid(raise_exception=True) + + # Don't allow a user to create many nomination entries in a single nomination + if NominationEntry.objects.filter( + nomination_id=nomination_filter[0].id, + actor__id=entry_serializer.validated_data["actor"].id + ).exists(): + raise ValidationError( + {'actor': ['This actor has already endorsed this nomination.']} + ) + + NominationEntry.objects.create(**entry_serializer.validated_data) + + data = NominationSerializer(nomination_filter[0]).data + + headers = self.get_success_headers(data) + return Response(data, status=status.HTTP_201_CREATED, headers=headers) def partial_update(self, request: HttpRequest, *args, **kwargs) -> Response: """ @@ -203,7 +258,7 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge elif instance.active and not data['active']: # 2. We're ending an active nomination. - if 'reason' in data: + if 'reason' in request.data: raise ValidationError( {'reason': ['This field cannot be set when ending a nomination.']} ) @@ -213,6 +268,11 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge {'end_reason': ['This field is required when ending a nomination.']} ) + if 'reviewed' in request.data: + raise ValidationError( + {'reviewed': ['This field cannot be set while you are ending a nomination.']} + ) + instance.ended_at = timezone.now() elif 'active' in data: @@ -221,6 +281,34 @@ class NominationViewSet(CreateModelMixin, RetrieveModelMixin, ListModelMixin, Ge {'active': ['This field can only be used to end a nomination']} ) + # This is actually covered, but for some reason coverage don't think so. + elif 'reviewed' in request.data: # pragma: no cover + # 4. We are altering the reviewed state of the nomination. + if not instance.active: + raise ValidationError( + {'reviewed': ['This field cannot be set if the nomination is inactive.']} + ) + + if 'reason' in request.data: + if 'actor' not in request.data: + raise ValidationError( + {'actor': ['This field is required when editing the reason.']} + ) + + entry_filter = NominationEntry.objects.filter( + nomination_id=instance.id, + actor__id=request.data['actor'] + ) + + if not entry_filter.exists(): + raise ValidationError( + {'actor': ["The actor doesn't exist or has not nominated the user."]} + ) + + entry = entry_filter[0] + entry.reason = request.data['reason'] + entry.save() + serializer.save() return Response(serializer.data) diff --git a/pydis_site/apps/api/viewsets/bot/user.py b/pydis_site/apps/api/viewsets/bot/user.py index 9571b3d7..25722f5a 100644 --- a/pydis_site/apps/api/viewsets/bot/user.py +++ b/pydis_site/apps/api/viewsets/bot/user.py @@ -1,21 +1,67 @@ +import typing +from collections import OrderedDict + +from django.core.exceptions import ObjectDoesNotExist +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.pagination import PageNumberPagination +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.serializers import ModelSerializer from rest_framework.viewsets import ModelViewSet -from rest_framework_bulk import BulkCreateModelMixin +from pydis_site.apps.api.models.bot.infraction import Infraction +from pydis_site.apps.api.models.bot.metricity import Metricity, NotFound from pydis_site.apps.api.models.bot.user import User from pydis_site.apps.api.serializers import UserSerializer -class UserViewSet(BulkCreateModelMixin, ModelViewSet): +class UserListPagination(PageNumberPagination): + """Custom pagination class for the User Model.""" + + page_size = 10000 + page_size_query_param = "page_size" + + def get_next_page_number(self) -> typing.Optional[int]: + """Get the next page number.""" + if not self.page.has_next(): + return None + page_number = self.page.next_page_number() + return page_number + + def get_previous_page_number(self) -> typing.Optional[int]: + """Get the previous page number.""" + if not self.page.has_previous(): + return None + + page_number = self.page.previous_page_number() + return page_number + + def get_paginated_response(self, data: list) -> Response: + """Override method to send modified response.""" + return Response(OrderedDict([ + ('count', self.page.paginator.count), + ('next_page_no', self.get_next_page_number()), + ('previous_page_no', self.get_previous_page_number()), + ('results', data) + ])) + + +class UserViewSet(ModelViewSet): """ View providing CRUD operations on Discord users through the bot. ## Routes ### GET /bot/users - Returns all users currently known. + Returns all users currently known with pagination. #### Response format - >>> [ - ... { + >>> { + ... 'count': 95000, + ... 'next_page_no': "2", + ... 'previous_page_no': None, + ... 'results': [ + ... { ... 'id': 409107086526644234, ... 'name': "Python", ... 'discriminator': 4329, @@ -26,8 +72,13 @@ class UserViewSet(BulkCreateModelMixin, ModelViewSet): ... 458226699344019457 ... ], ... 'in_guild': True - ... } - ... ] + ... }, + ... ] + ... } + + #### Optional Query Parameters + - page_size: number of Users in one page, defaults to 10,000 + - page: page number #### Status codes - 200: returned on success @@ -53,9 +104,41 @@ class UserViewSet(BulkCreateModelMixin, ModelViewSet): - 200: returned on success - 404: if a user with the given `snowflake` could not be found + ### GET /bot/users/<snowflake:int>/metricity_data + Gets metricity data for a single user by ID. + + #### Response format + >>> { + ... "joined_at": "2020-10-06T21:54:23.540766", + ... "total_messages": 2, + ... "voice_banned": False, + ... "activity_blocks": 1 + ...} + + #### Status codes + - 200: returned on success + - 404: if a user with the given `snowflake` could not be found + + ### GET /bot/users/<snowflake:int>/metricity_review_data + Gets metricity data for a single user's review by ID. + + #### Response format + >>> { + ... 'joined_at': '2020-08-26T08:09:43.507000', + ... 'top_channel_activity': [['off-topic', 15], + ... ['talent-pool', 4], + ... ['defcon', 2]], + ... 'total_messages': 22 + ... } + + #### Status codes + - 200: returned on success + - 404: if a user with the given `snowflake` could not be found + ### POST /bot/users Adds a single or multiple new users. The roles attached to the user(s) must be roles known by the site. + Users that already exist in the database will be skipped. #### Request body >>> { @@ -67,11 +150,13 @@ class UserViewSet(BulkCreateModelMixin, ModelViewSet): ... } Alternatively, request users can be POSTed as a list of above objects, - in which case multiple users will be created at once. + in which case multiple users will be created at once. In this case, + the response is an empty list. #### Status codes - 201: returned on success - 400: if one of the given roles does not exist, or one of the given fields is invalid + - 400: if multiple user objects with the same id are given ### PUT /bot/users/<snowflake:int> Update the user with the given `snowflake`. @@ -109,6 +194,34 @@ class UserViewSet(BulkCreateModelMixin, ModelViewSet): - 400: if the request body was invalid, see response body for details - 404: if the user with the given `snowflake` could not be found + ### BULK PATCH /bot/users/bulk_patch + Update users with the given `ids` and `details`. + `id` field and at least one other field is mandatory. + + #### Request body + >>> [ + ... { + ... 'id': int, + ... 'name': str, + ... 'discriminator': int, + ... 'roles': List[int], + ... 'in_guild': bool + ... }, + ... { + ... 'id': int, + ... 'name': str, + ... 'discriminator': int, + ... 'roles': List[int], + ... 'in_guild': bool + ... }, + ... ] + + #### Status codes + - 200: returned on success + - 400: if the request body was invalid, see response body for details + - 400: if multiple user objects with the same id are given + - 404: if the user with the given id does not exist + ### DELETE /bot/users/<snowflake:int> Deletes the user with the given `snowflake`. @@ -118,4 +231,65 @@ class UserViewSet(BulkCreateModelMixin, ModelViewSet): """ serializer_class = UserSerializer - queryset = User.objects + queryset = User.objects.all().order_by("id") + pagination_class = UserListPagination + + def get_serializer(self, *args, **kwargs) -> ModelSerializer: + """Set Serializer many attribute to True if request body contains a list.""" + if isinstance(kwargs.get('data', {}), list): + kwargs['many'] = True + + return super().get_serializer(*args, **kwargs) + + @action(detail=False, methods=["PATCH"], name='user-bulk-patch') + def bulk_patch(self, request: Request) -> Response: + """Update multiple User objects in a single request.""" + serializer = self.get_serializer( + instance=self.get_queryset(), + data=request.data, + many=True, + partial=True + ) + + serializer.is_valid(raise_exception=True) + serializer.save() + + return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=True) + def metricity_data(self, request: Request, pk: str = None) -> Response: + """Request handler for metricity_data endpoint.""" + user = self.get_object() + + try: + Infraction.objects.get(user__id=user.id, active=True, type="voice_ban") + except ObjectDoesNotExist: + voice_banned = False + else: + voice_banned = True + + with Metricity() as metricity: + try: + data = metricity.user(user.id) + data["total_messages"] = metricity.total_messages(user.id) + data["voice_banned"] = voice_banned + data["activity_blocks"] = metricity.total_message_blocks(user.id) + return Response(data, status=status.HTTP_200_OK) + except NotFound: + return Response(dict(detail="User not found in metricity"), + status=status.HTTP_404_NOT_FOUND) + + @action(detail=True) + def metricity_review_data(self, request: Request, pk: str = None) -> Response: + """Request handler for metricity_review_data endpoint.""" + user = self.get_object() + + with Metricity() as metricity: + try: + data = metricity.user(user.id) + data["total_messages"] = metricity.total_messages(user.id) + data["top_channel_activity"] = metricity.top_channel_activity(user.id) + return Response(data, status=status.HTTP_200_OK) + except NotFound: + return Response(dict(detail="User not found in metricity"), + status=status.HTTP_404_NOT_FOUND) diff --git a/pydis_site/apps/api/viewsets/log_entry.py b/pydis_site/apps/api/viewsets/log_entry.py deleted file mode 100644 index 9108a4fa..00000000 --- a/pydis_site/apps/api/viewsets/log_entry.py +++ /dev/null @@ -1,36 +0,0 @@ -from rest_framework.mixins import CreateModelMixin -from rest_framework.viewsets import GenericViewSet - -from pydis_site.apps.api.models.log_entry import LogEntry -from pydis_site.apps.api.serializers import LogEntrySerializer - - -class LogEntryViewSet(CreateModelMixin, GenericViewSet): - """ - View supporting the creation of log entries in the database for viewing via the log browser. - - ## Routes - ### POST /logs - Create a new log entry. - - #### Request body - >>> { - ... 'application': str, # 'bot' | 'seasonalbot' | 'site' - ... 'logger_name': str, # such as 'bot.cogs.moderation' - ... 'timestamp': Optional[str], # from `datetime.utcnow().isoformat()` - ... 'level': str, # 'debug' | 'info' | 'warning' | 'error' | 'critical' - ... 'module': str, # such as 'pydis_site.apps.api.serializers' - ... 'line': int, # > 0 - ... 'message': str, # textual formatted content of the logline - ... } - - #### Status codes - - 201: returned on success - - 400: if the request body has invalid fields, see the response for details - - ## Authentication - Requires a API token. - """ - - queryset = LogEntry.objects.all() - serializer_class = LogEntrySerializer diff --git a/pydis_site/apps/home/migrations/0002_auto_now_on_repository_metadata.py b/pydis_site/apps/home/migrations/0002_auto_now_on_repository_metadata.py new file mode 100644 index 00000000..7e78045b --- /dev/null +++ b/pydis_site/apps/home/migrations/0002_auto_now_on_repository_metadata.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.11 on 2020-12-21 22:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='repositorymetadata', + name='last_updated', + field=models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.'), + ), + ] diff --git a/pydis_site/apps/home/models/repository_metadata.py b/pydis_site/apps/home/models/repository_metadata.py index 92d2404d..00a83cd7 100644 --- a/pydis_site/apps/home/models/repository_metadata.py +++ b/pydis_site/apps/home/models/repository_metadata.py @@ -1,32 +1,31 @@ from django.db import models -from django.utils import timezone class RepositoryMetadata(models.Model): """Information about one of our repos fetched from the GitHub API.""" last_updated = models.DateTimeField( - default=timezone.now, - help_text="The date and time this data was last fetched." + help_text="The date and time this data was last fetched.", + auto_now=True, ) repo_name = models.CharField( primary_key=True, max_length=40, - help_text="The full name of the repo, e.g. python-discord/site" + help_text="The full name of the repo, e.g. python-discord/site", ) description = models.CharField( max_length=400, - help_text="The description of the repo." + help_text="The description of the repo.", ) forks = models.IntegerField( - help_text="The number of forks of this repo" + help_text="The number of forks of this repo", ) stargazers = models.IntegerField( - help_text="The number of stargazers for this repo" + help_text="The number of stargazers for this repo", ) language = models.CharField( max_length=20, - help_text="The primary programming language used for this repo." + help_text="The primary programming language used for this repo.", ) def __str__(self): diff --git a/pydis_site/apps/home/tests/mock_github_api_response.json b/pydis_site/apps/home/tests/mock_github_api_response.json index 10be4f99..ddbffed8 100644 --- a/pydis_site/apps/home/tests/mock_github_api_response.json +++ b/pydis_site/apps/home/tests/mock_github_api_response.json @@ -35,7 +35,7 @@ "forks_count": 31 }, { - "full_name": "python-discord/seasonalbot", + "full_name": "python-discord/sir-lancebot", "description": "test", "stargazers_count": 97, "language": "Python", diff --git a/pydis_site/apps/home/tests/test_repodata_helpers.py b/pydis_site/apps/home/tests/test_repodata_helpers.py index 77b1a68d..5634bc9b 100644 --- a/pydis_site/apps/home/tests/test_repodata_helpers.py +++ b/pydis_site/apps/home/tests/test_repodata_helpers.py @@ -123,10 +123,38 @@ class TestRepositoryMetadataHelpers(TestCase): mock_get.return_value.json.return_value = ['garbage'] metadata = self.home_view._get_repo_data() - self.assertEquals(len(metadata), len(self.home_view.repos)) - for item in metadata: - with self.subTest(item=item): - self.assertEqual(item.description, "Not available.") - self.assertEqual(item.forks, 999) - self.assertEqual(item.stargazers, 999) - self.assertEqual(item.language, "Python") + self.assertEquals(len(metadata), 0) + + def test_cleans_up_stale_metadata(self): + """Tests that we clean up stale metadata when we start the HomeView.""" + repo_data = RepositoryMetadata( + repo_name="python-discord/INVALID", + description="testrepo", + forks=42, + stargazers=42, + language="English", + last_updated=timezone.now() - timedelta(seconds=HomeView.repository_cache_ttl + 1), + ) + repo_data.save() + self.home_view.__init__() + cached_repos = RepositoryMetadata.objects.all() + cached_names = [repo.repo_name for repo in cached_repos] + + self.assertNotIn("python-discord/INVALID", cached_names) + + def test_dont_clean_up_unstale_metadata(self): + """Tests that we don't clean up good metadata when we start the HomeView.""" + repo_data = RepositoryMetadata( + repo_name="python-discord/site", + description="testrepo", + forks=42, + stargazers=42, + language="English", + last_updated=timezone.now() - timedelta(seconds=HomeView.repository_cache_ttl + 1), + ) + repo_data.save() + self.home_view.__init__() + cached_repos = RepositoryMetadata.objects.all() + cached_names = [repo.repo_name for repo in cached_repos] + + self.assertIn("python-discord/site", cached_names) diff --git a/pydis_site/apps/home/urls.py b/pydis_site/apps/home/urls.py index e475c491..1e2af8f3 100644 --- a/pydis_site/apps/home/urls.py +++ b/pydis_site/apps/home/urls.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.urls import include, path -from .views import HomeView +from .views import HomeView, timeline app_name = 'home' urlpatterns = [ @@ -11,4 +11,5 @@ urlpatterns = [ path('resources/', include('pydis_site.apps.resources.urls')), path('pages/', include('pydis_site.apps.content.urls')), path('events/', include('pydis_site.apps.events.urls', namespace='events')), + path('timeline/', timeline, name="timeline"), ] diff --git a/pydis_site/apps/home/views/__init__.py b/pydis_site/apps/home/views/__init__.py index 971d73a3..28cc4d65 100644 --- a/pydis_site/apps/home/views/__init__.py +++ b/pydis_site/apps/home/views/__init__.py @@ -1,3 +1,3 @@ -from .home import HomeView +from .home import HomeView, timeline -__all__ = ["HomeView"] +__all__ = ["HomeView", "timeline"] diff --git a/pydis_site/apps/home/views/home.py b/pydis_site/apps/home/views/home.py index 3b5cd5ac..e77772fb 100644 --- a/pydis_site/apps/home/views/home.py +++ b/pydis_site/apps/home/views/home.py @@ -1,4 +1,4 @@ -import datetime +import logging from typing import Dict, List import requests @@ -10,11 +10,13 @@ from django.views import View from pydis_site.apps.home.models import RepositoryMetadata +log = logging.getLogger(__name__) + class HomeView(View): """The main landing page for the website.""" - github_api = "https://api.github.com/users/python-discord/repos" + github_api = "https://api.github.com/users/python-discord/repos?per_page=100" repository_cache_ttl = 3600 # Which of our GitHub repos should be displayed on the front page, and in which order? @@ -22,82 +24,98 @@ class HomeView(View): "python-discord/site", "python-discord/bot", "python-discord/snekbox", - "python-discord/seasonalbot", + "python-discord/sir-lancebot", "python-discord/metricity", "python-discord/django-simple-bulma", ] + def __init__(self): + """Clean up stale RepositoryMetadata.""" + RepositoryMetadata.objects.exclude(repo_name__in=self.repos).delete() + def _get_api_data(self) -> Dict[str, Dict[str, str]]: - """Call the GitHub API and get information about our repos.""" - repo_dict: Dict[str, dict] = {repo_name: {} for repo_name in self.repos} + """ + Call the GitHub API and get information about our repos. + + If we're unable to get that info for any reason, return an empty dict. + """ + repo_dict = {} # Fetch the data from the GitHub API api_data: List[dict] = requests.get(self.github_api).json() # Process the API data into our dict for repo in api_data: - full_name = repo["full_name"] - - if full_name in self.repos: - repo_dict[full_name] = { - "full_name": repo["full_name"], - "description": repo["description"], - "language": repo["language"], - "forks_count": repo["forks_count"], - "stargazers_count": repo["stargazers_count"], - } + try: + full_name = repo["full_name"] + + if full_name in self.repos: + repo_dict[full_name] = { + "full_name": repo["full_name"], + "description": repo["description"], + "language": repo["language"], + "forks_count": repo["forks_count"], + "stargazers_count": repo["stargazers_count"], + } + # Something is not right about the API data we got back from GitHub. + except (TypeError, ConnectionError, KeyError) as e: + log.error( + "Unable to parse the GitHub repository metadata from response!", + extra={ + 'api_data': api_data, + 'error': e + } + ) + continue return repo_dict def _get_repo_data(self) -> List[RepositoryMetadata]: """Build a list of RepositoryMetadata objects that we can use to populate the front page.""" - # Try to get site data from the cache - try: - repo_data = RepositoryMetadata.objects.get(repo_name="python-discord/site") + database_repositories = [] - # If the data is stale, we should refresh it. - if (timezone.now() - repo_data.last_updated).seconds > self.repository_cache_ttl: + # First, let's see if we have any metadata cached. + cached_data = RepositoryMetadata.objects.all() - # Try to get new data from the API. If it fails, return the cached data. - try: - api_repositories = self._get_api_data() - except (TypeError, ConnectionError): - return RepositoryMetadata.objects.all() - database_repositories = [] - - # Update or create all RepoData objects in self.repos - for repo_name, api_data in api_repositories.items(): - try: - repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) - repo_data.description = api_data["description"] - repo_data.language = api_data["language"] - repo_data.forks = api_data["forks_count"] - repo_data.stargazers = api_data["stargazers_count"] - except RepositoryMetadata.DoesNotExist: - repo_data = RepositoryMetadata( - repo_name=api_data["full_name"], - description=api_data["description"], - forks=api_data["forks_count"], - stargazers=api_data["stargazers_count"], - language=api_data["language"], - ) - repo_data.save() - database_repositories.append(repo_data) - return database_repositories - - # Otherwise, if the data is fresher than 2 minutes old, we should just return it. - else: - return RepositoryMetadata.objects.all() + # If we don't, we have to create some! + if not cached_data: - # If this is raised, the database has no repodata at all, we will create them all. - except RepositoryMetadata.DoesNotExist: - database_repositories = [] - try: - # Get new data from API - api_repositories = self._get_api_data() + # Try to get new data from the API. If it fails, we'll return an empty list. + # In this case, we simply don't display our projects on the site. + api_repositories = self._get_api_data() + + # Create all the repodata records in the database. + for api_data in api_repositories.values(): + repo_data = RepositoryMetadata( + repo_name=api_data["full_name"], + description=api_data["description"], + forks=api_data["forks_count"], + stargazers=api_data["stargazers_count"], + language=api_data["language"], + ) + + repo_data.save() + database_repositories.append(repo_data) + + return database_repositories + + # If the data is stale, we should refresh it. + if (timezone.now() - cached_data[0].last_updated).seconds > self.repository_cache_ttl: + # Try to get new data from the API. If it fails, return the cached data. + api_repositories = self._get_api_data() + + if not api_repositories: + return RepositoryMetadata.objects.all() - # Create all the repodata records in the database. - for api_data in api_repositories.values(): + # Update or create all RepoData objects in self.repos + for repo_name, api_data in api_repositories.items(): + try: + repo_data = RepositoryMetadata.objects.get(repo_name=repo_name) + repo_data.description = api_data["description"] + repo_data.language = api_data["language"] + repo_data.forks = api_data["forks_count"] + repo_data.stargazers = api_data["stargazers_count"] + except RepositoryMetadata.DoesNotExist: repo_data = RepositoryMetadata( repo_name=api_data["full_name"], description=api_data["description"], @@ -105,24 +123,20 @@ class HomeView(View): stargazers=api_data["stargazers_count"], language=api_data["language"], ) - repo_data.save() - database_repositories.append(repo_data) - except TypeError: - for repo_name in self.repos: - repo_data = RepositoryMetadata( - last_updated=timezone.now() - datetime.timedelta(minutes=50), - repo_name=repo_name, - description="Not available.", - forks=999, - stargazers=999, - language="Python", - ) - repo_data.save() - database_repositories.append(repo_data) - + repo_data.save() + database_repositories.append(repo_data) return database_repositories + # Otherwise, if the data is fresher than 2 minutes old, we should just return it. + else: + return RepositoryMetadata.objects.all() + def get(self, request: WSGIRequest) -> HttpResponse: """Collect repo data and render the homepage view.""" repo_data = self._get_repo_data() return render(request, "home/index.html", {"repo_data": repo_data}) + + +def timeline(request: WSGIRequest) -> HttpResponse: + """Render timeline view.""" + return render(request, 'home/timeline.html') diff --git a/pydis_site/apps/redirect/tests.py b/pydis_site/apps/redirect/tests.py index c145ecda..fce2642f 100644 --- a/pydis_site/apps/redirect/tests.py +++ b/pydis_site/apps/redirect/tests.py @@ -40,11 +40,14 @@ class RedirectTests(TestCase): if data.get("prefix_redirect", False): expected_args = ( "".join( - tuple(data.get("redirect_arguments", ())) + TESTING_ARGUMENTS.get(name, ()) + tuple(data.get("redirect_arguments", ())) + + TESTING_ARGUMENTS.get(name, ()) ), ) else: - expected_args = TESTING_ARGUMENTS.get(name, ()) + tuple(data.get("redirect_arguments", ())) + expected_args = ( + TESTING_ARGUMENTS.get(name, ()) + tuple(data.get("redirect_arguments", ())) + ) self.assertEqual(1, len(resp.redirect_chain)) self.assertRedirects( diff --git a/pydis_site/apps/resources/resources/reading/books/effective_python.yaml b/pydis_site/apps/resources/resources/reading/books/effective_python.yaml index 1d4124cc..becd0578 100644 --- a/pydis_site/apps/resources/resources/reading/books/effective_python.yaml +++ b/pydis_site/apps/resources/resources/reading/books/effective_python.yaml @@ -1,4 +1,4 @@ -description: A book that gives 59 best practices for writing excellent Python. Great +description: A book that gives 90 best practices for writing excellent Python. Great for intermediates. name: Effective Python position: 3 @@ -7,8 +7,9 @@ urls: url: https://effectivepython.com/ color: teal - icon: branding/amazon - url: https://www.amazon.com/Effective-Python-Specific-Software-Development/dp/0134034287 + url: https://www.amazon.com/Effective-Python-Specific-Software-Development/dp/0134853989 color: amazon-orange + title: Amazon - icon: branding/github url: https://github.com/bslatkin/effectivepython color: black |