From cc156c2f1b880a250b8011dd97c7812c379b2e7c Mon Sep 17 00:00:00 2001 From: Momo Date: Sat, 21 Jul 2018 20:59:49 +0000 Subject: Infraction system API to replace Rowboat --- pysite/decorators.py | 4 +- pysite/tables.py | 15 + pysite/views/api/bot/infractions.py | 545 ++++++++++++++++++++++++++++++++++++ 3 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 pysite/views/api/bot/infractions.py (limited to 'pysite') diff --git a/pysite/decorators.py b/pysite/decorators.py index de914c6f..fbfb90f8 100644 --- a/pysite/decorators.py +++ b/pysite/decorators.py @@ -141,8 +141,8 @@ def api_params( try: schema.validate(data) - except SchemaError: - return self.error(ErrorCodes.incorrect_parameters) + except SchemaError as e: + return self.error(ErrorCodes.incorrect_parameters, str(e)) return f(self, data, *args, **kwargs) diff --git a/pysite/tables.py b/pysite/tables.py index 8f849664..3485065f 100644 --- a/pysite/tables.py +++ b/pysite/tables.py @@ -259,6 +259,21 @@ TABLES = { ]) ), + "bot_infractions": Table( + primary_key="id", + keys=sorted([ + "id", # str + "user_id", # str + "actor_id", # str + "reason", # str + "type" # str + "inserted_at", # datetime + "expires_at", # datetime + "active", # bool + "legacy_rowboat_id" # str + ]) + ), + "watched_users": Table( # Users being monitored by the bot's BigBrother cog primary_key="user_id", keys=sorted([ diff --git a/pysite/views/api/bot/infractions.py b/pysite/views/api/bot/infractions.py new file mode 100644 index 00000000..8b92156a --- /dev/null +++ b/pysite/views/api/bot/infractions.py @@ -0,0 +1,545 @@ +""" +INFRACTIONS API + +"GET" endpoints in this API may take the following optional parameters, depending on the endpoint: + - active: filters infractions that are active (true), expired (false), or either (not present/any) + - expand: expands the result data with the information about the users (slower) + +Infraction Schema: + This schema is used when an infraction's data is returned. + + Root object: + "id" (str): the UUID of the infraction. + "inserted_at" (str): the date and time of the creation of this infraction (RFC1123 format). + "expires_at" (str): the date and time of the expiration of this infraction (RC1123 format), may be null. + The significance of this field being null depends on the type of infraction. Duration-based infractions + have a "null" expiration if they are permanent. Other infraction types do not have expirations. + "active" (bool): whether the infraction is still active. Note that the check for expiration of + duration-based infractions is done by the API, so you should check for expiration using this "active" field. + "user" (object): the user to which the infraction was applied. + "user_id" (str): the Discord ID of the user. + "username" (optional str): the username of the user. This field is only present if the query was expanded. + "discriminator" (optional int): the username discriminator of the user. This field is only present if the + query was expanded. + "avatar" (optional str): the avatar URL of the user. This field is only present if the query was expanded. + "actor" (object): the user which applied the infraction. + This object uses the same schema as the "user" field. + "type" (str): the type of the infraction. + "reason" (str): the reason for the infraction. + + +Endpoints: + + GET /bot/infractions + Gets a list of all infractions, regardless of type or user. + Parameters: "active", "expand". + This endpoint returns an array of infraction objects. + + GET /bot/infractions/user/ + Gets a list of all infractions for a user. + Parameters: "active", "expand". + This endpoint returns an array of infraction objects. + + GET /bot/infractions/type/ + Gets a list of all infractions of the given type (ban, mute, etc.) + Parameters: "active", "expand". + This endpoint returns an array of infraction objects. + + GET /bot/infractions/user// + Gets a list of all infractions of the given type for a user. + Parameters: "active", "expand". + This endpoint returns an array of infraction objects. + + GET /bot/infractions/user///current + Gets the active infraction (if any) of the given type for a user. + Parameters: "expand". + This endpoint returns an object with the "infraction" key, which is either set to null (no infraction) + or the query's corresponding infraction. It will not return an infraction if the type of the infraction + isn't duration-based (e.g. kick, warning, etc.) + + GET /bot/infractions/id/ + Gets the infraction (if any) for the given ID. + Parameters: "expand". + This endpoint returns an object with the "infraction" key, which is either set to null (no infraction) + or the infraction corresponding to the ID. + + POST /bot/infractions + Creates an infraction for a user. + Parameters (JSON payload): + "type" (str): the type of the infraction (must be a valid infraction type). + "reason" (str): the reason of the infraction. + "user_id" (str): the Discord ID of the user who is being given the infraction. + "actor_id" (str): the Discord ID of the user who submitted the infraction. + "duration" (optional str): the duration of the infraction. This is ignored for infractions + which are not duration-based. For other infraction types, omitting this field may imply permanence. + "expand" (optional bool): whether to expand the infraction user data once the infraction is inserted and returned. + + PATCH /bot/infractions + Updates an infractions. + Parameters (JSON payload): + "id" (str): the ID of the infraction to update. + "reason" (optional str): if provided, the new reason for the infraction. + "duration" (optional str): if provided, updates the expiration of the infraction to the time of UPDATING + plus the duration. If set to null, the expiration is also set to null (may imply permanence). + "active" (optional bool): if provided, activates or deactivates the infraction. This does not do anything + if the infraction isn't duration-based, or if the infraction has already expired. + "expand" (optional bool): whether to expand the infraction user data once the infraction is updated and returned. +""" + +import datetime +from typing import NamedTuple + +import rethinkdb +from flask import jsonify +from schema import Optional, Or, Schema + +from pysite.base_route import APIView +from pysite.constants import ErrorCodes, ValidationTypes +from pysite.decorators import api_key, api_params +from pysite.mixins import DBMixin +from pysite.utils.time import parse_duration + + +class InfractionType(NamedTuple): + timed_infraction: bool # whether the infraction is active until it expires. + + +RFC1123_FORMAT = "%a, %d %b %Y %H:%M:%S GMT" + +INFRACTION_TYPES = { + "warning": InfractionType(timed_infraction=False), + "mute": InfractionType(timed_infraction=True), + "ban": InfractionType(timed_infraction=True), + "kick": InfractionType(timed_infraction=False), + "superstar": InfractionType(timed_infraction=True) # hiphopify +} + +GET_SCHEMA = Schema({ + Optional("active"): str, + Optional("expand"): str +}) + +GET_ACTIVE_SCHEMA = Schema({ + Optional("expand"): str +}) + +CREATE_INFRACTION_SCHEMA = Schema({ + "type": lambda tp: tp in INFRACTION_TYPES, + "reason": Or(str, None), + "user_id": str, # Discord user ID + "actor_id": str, # Discord user ID + Optional("duration"): str, # If not provided, may imply permanence depending on the infraction + Optional("expand"): bool +}) + +UPDATE_INFRACTION_SCHEMA = Schema({ + "id": str, + Optional("reason"): Or(str, None), + Optional("duration"): Or(str, None), + Optional("active"): bool +}) + +IMPORT_INFRACTIONS_SCHEMA = Schema([ + { + "id": str, + "active": bool, + "actor": { + "id": str + }, + "created_at": str, + "expires_at": Or(str, None), + "reason": Or(str, None), + "type": { + "name": str + }, + "user": { + "id": str + } + } +], ignore_extra_keys=True) + + +class InfractionsView(APIView, DBMixin): + path = "/bot/infractions" + name = "bot.infractions" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params=None): + return _infraction_list_filtered(self, params, {}) + + @api_key + @api_params(schema=CREATE_INFRACTION_SCHEMA, validation_type=ValidationTypes.json) + def post(self, data): + deactivate_infraction_query = None + + infraction_type = data["type"] + user_id = data["user_id"] + actor_id = data["actor_id"] + reason = data["reason"] + duration_str = data.get("duration") + expand = data.get("expand") + expires_at = None + inserted_at = datetime.datetime.now(tz=datetime.timezone.utc) + + if infraction_type not in INFRACTION_TYPES: + return self.error(ErrorCodes.incorrect_parameters, "Invalid infraction type.") + + # check if the user already has an active infraction of this type + # if so, we need to disable that infraction and create a new infraction + if INFRACTION_TYPES[infraction_type].timed_infraction: + active_infraction_query = \ + self.db.query(self.table_name).merge(_merge_active_check()) \ + .filter({"user_id": user_id, "type": infraction_type, "active": True}) \ + .limit(1).nth(0).default(None) + + active_infraction = self.db.run(active_infraction_query) + if active_infraction: + deactivate_infraction_query = \ + self.db.query(self.table_name) \ + .get(active_infraction["id"]) \ + .update({"active": False}) + + if duration_str: + try: + expires_at = parse_duration(duration_str) + except ValueError: + return self.error( + ErrorCodes.incorrect_parameters, + "Invalid duration format." + ) + + infraction_insert_doc = { + "actor_id": actor_id, + "user_id": user_id, + "type": infraction_type, + "reason": reason, + "inserted_at": inserted_at, + "expires_at": expires_at + } + + infraction_id = self.db.insert(self.table_name, infraction_insert_doc)["generated_keys"][0] + + if deactivate_infraction_query: + self.db.run(deactivate_infraction_query) + + query = self.db.query(self.table_name).get(infraction_id) \ + .merge(_merge_expand_users(self, expand)) \ + .merge(_merge_active_check()) \ + .without("user_id", "actor_id").default(None) + return jsonify({ + "infraction": self.db.run(query) + }) + + @api_key + @api_params(schema=UPDATE_INFRACTION_SCHEMA, validation_type=ValidationTypes.json) + def patch(self, data): + expand = data.get("expand") + update_collection = { + "id": data["id"] + } + + if "reason" in data: + update_collection["reason"] = data["reason"] + + if "active" in data: + update_collection["active"] = data["active"] + + if "duration" in data: + duration_str = data["duration"] + if duration_str is None: + update_collection["expires_at"] = None + else: + try: + update_collection["expires_at"] = parse_duration(duration_str) + except ValueError: + return self.error( + ErrorCodes.incorrect_parameters, + "Invalid duration format." + ) + + query_update = self.db.query(self.table_name).update(update_collection) + result_update = self.db.run(query_update) + + if not result_update["replaced"]: + return jsonify({ + "success": False + }) + + # return the updated infraction + query = self.db.query(self.table_name).get(data["id"]) \ + .merge(_merge_expand_users(self, expand)) \ + .merge(_merge_active_check()) \ + .without("user_id", "actor_id").default(None) + infraction = self.db.run(query) + + return jsonify({ + "infraction": infraction, + "success": True + }) + + +class InfractionById(APIView, DBMixin): + path = "/bot/infractions/id/" + name = "bot.infractions.id" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_ACTIVE_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params, infraction_id): + params = params or {} + expand = parse_bool(params.get("expand"), default=False) + + query = self.db.query(self.table_name).get(infraction_id) \ + .merge(_merge_expand_users(self, expand)) \ + .merge(_merge_active_check()) \ + .without("user_id", "actor_id").default(None) + return jsonify({ + "infraction": self.db.run(query) + }) + + +class ListInfractionsByUserView(APIView, DBMixin): + path = "/bot/infractions/user/" + name = "bot.infractions.user" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params, user_id): + return _infraction_list_filtered(self, params, { + "user_id": user_id + }) + + +class ListInfractionsByTypeView(APIView, DBMixin): + path = "/bot/infractions/type/" + name = "bot.infractions.type" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params, type): + return _infraction_list_filtered(self, params, { + "type": type + }) + + +class ListInfractionsByTypeAndUserView(APIView, DBMixin): + path = "/bot/infractions/user//" + name = "bot.infractions.user.type" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params, user_id, type): + return _infraction_list_filtered(self, params, { + "user_id": user_id, + "type": type + }) + + +class CurrentInfractionByTypeAndUserView(APIView, DBMixin): + path = "/bot/infractions/user///current" + name = "bot.infractions.user.type.current" + table_name = "bot_infractions" + + @api_key + @api_params(schema=GET_ACTIVE_SCHEMA, validation_type=ValidationTypes.params) + def get(self, params, user_id, infraction_type): + params = params or {} + expand = parse_bool(params.get("expand"), default=False) + + query_filter = { + "user_id": user_id, + "type": infraction_type + } + query = _merged_query(self, expand, query_filter).filter({ + "active": True + }).order_by(rethinkdb.desc("data")).limit(1).nth(0).default(None) + return jsonify({ + "infraction": self.db.run(query) + }) + + +class ImportRowboatInfractionsView(APIView, DBMixin): + path = "/bot/infractions/import" + name = "bot.infractions.import" + table_name = "bot_infractions" + + @api_key + @api_params(schema=IMPORT_INFRACTIONS_SCHEMA, validation_type=ValidationTypes.json) + def post(self, data): + # keep track of the un-bans, to apply after the import is complete. + unbans = [] + infractions = [] + + # previously imported infractions + imported_infractions = self.db.run( + self.db.query(self.table_name).filter( + lambda row: row.has_fields("legacy_rowboat_id") + ).fold([], lambda acc, row: acc.append(row["legacy_rowboat_id"])).coerce_to("array") + ) + + for rowboat_infraction_data in data: + legacy_rowboat_id = rowboat_infraction_data["id"] + if legacy_rowboat_id in imported_infractions: + continue + infraction_type = rowboat_infraction_data["type"]["name"] + if infraction_type == "unban": + unbans.append(rowboat_infraction_data) + continue + # adjust infraction types + if infraction_type == "tempmute": + infraction_type = "mute" + if infraction_type == "tempban": + infraction_type = "ban" + if infraction_type not in INFRACTION_TYPES: + # unknown infraction type + continue + active = rowboat_infraction_data["active"] + reason = rowboat_infraction_data["reason"] or "" + user_id = rowboat_infraction_data["user"]["id"] + actor_id = rowboat_infraction_data["actor"]["id"] + inserted_at_str = rowboat_infraction_data["created_at"] + try: + inserted_at = parse_rfc1123(inserted_at_str) + except ValueError: + continue + expires_at_str = rowboat_infraction_data["expires_at"] + if expires_at_str is not None: + try: + expires_at = parse_rfc1123(expires_at_str) + except ValueError: + continue + else: + expires_at = None + infractions.append({ + "legacy_rowboat_id": legacy_rowboat_id, + "active": active, + "reason": reason, + "user_id": user_id, + "actor_id": actor_id, + "inserted_at": inserted_at, + "expires_at": expires_at, + "type": infraction_type + }) + + insertion_query = self.db.query(self.table_name).insert(infractions) + inserted_count = self.db.run(insertion_query)["inserted"] + + # apply unbans + for unban_data in unbans: + inserted_at_str = unban_data["created_at"] + user_id = unban_data["user"]["id"] + try: + inserted_at = parse_rfc1123(inserted_at_str) + except ValueError: + continue + self.db.run( + self.db.query(self.table_name).filter( + lambda row: (row["user_id"].eq(user_id)) & + (row["type"].eq("ban")) & + (row["inserted_at"] < inserted_at) + ).pluck("id").merge(lambda row: { + "active": False + }).coerce_to("array").for_each(lambda doc: self.db.query(self.table_name).get(doc["id"]).update(doc)) + ) + + return jsonify({ + "success": True, + "inserted_count": inserted_count + }) + + +def _infraction_list_filtered(view, params=None, query_filter=None): + params = params or {} + query_filter = query_filter or {} + active = parse_bool(params.get("active")) + expand = parse_bool(params.get("expand"), default=False) + + if active is not None: + query_filter["active"] = active + + query = _merged_query(view, expand, query_filter) + return jsonify(view.db.run(query.coerce_to("array"))) + + +def _merged_query(view, expand, query_filter): + return view.db.query(view.table_name).merge(_merge_active_check()).filter(query_filter) \ + .merge(_merge_expand_users(view, expand)).without("user_id", "actor_id") + + +def _merge_active_check(): + # Checks if the "active" field has been set to false (manual infraction removal). + # If not, the "active" field is set to whether the infraction has expired. + def _merge(row): + return { + "active": + rethinkdb.branch( + _is_timed_infraction(row["type"]), + rethinkdb.branch( + row["active"].default(True).eq(False), + False, + rethinkdb.branch( + row["expires_at"].eq(None), + True, + row["expires_at"] > rethinkdb.now() + ) + ), + False + ) + } + + return _merge + + +def _merge_expand_users(view, expand): + def _do_expand(user_id): + if not user_id: + return None + # Expands the user information, if it is in the database. + + if expand: + return view.db.query("users").get(user_id).default({ + "user_id": user_id + }) + + return { + "user_id": user_id + } + + def _merge(row): + return { + "user": _do_expand(row["user_id"].default(None)), + "actor": _do_expand(row["actor_id"].default(None)) + } + + return _merge + + +def _is_timed_infraction(type_var): + # this method generates an ReQL expression to check if the given type + # is a "timed infraction" (i.e it can expire or be permanent) + + timed_infractions = filter(lambda key: INFRACTION_TYPES[key].timed_infraction, INFRACTION_TYPES.keys()) + expr = rethinkdb.expr(False) + for infra_type in timed_infractions: + expr = expr | type_var.eq(infra_type) + return expr + + +def parse_rfc1123(time_str): + return datetime.datetime.strptime(time_str, RFC1123_FORMAT).replace(tzinfo=datetime.timezone.utc) + + +def parse_bool(a_string, default=None): + # Not present, null or any: returns default (defaults to None) + # false, no, or 0: returns False + # anything else: True + if a_string is None or a_string == "null" or a_string == "any": + return default + if a_string.lower() == "false" or a_string.lower() == "no" or a_string == "0": + return False + return True -- cgit v1.2.3 From 6ee29c01d19aa64b29bcc333f55c3d147380e49f Mon Sep 17 00:00:00 2001 From: Momo Date: Thu, 26 Jul 2018 22:16:02 +0000 Subject: Tweaks to the Infractions API for the bot --- pysite/tables.py | 2 +- pysite/views/api/bot/infractions.py | 61 ++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 18 deletions(-) (limited to 'pysite') diff --git a/pysite/tables.py b/pysite/tables.py index 3485065f..838d69f9 100644 --- a/pysite/tables.py +++ b/pysite/tables.py @@ -269,7 +269,7 @@ TABLES = { "type" # str "inserted_at", # datetime "expires_at", # datetime - "active", # bool + "closed", # bool "legacy_rowboat_id" # str ]) ), diff --git a/pysite/views/api/bot/infractions.py b/pysite/views/api/bot/infractions.py index 8b92156a..eee40b82 100644 --- a/pysite/views/api/bot/infractions.py +++ b/pysite/views/api/bot/infractions.py @@ -4,6 +4,8 @@ INFRACTIONS API "GET" endpoints in this API may take the following optional parameters, depending on the endpoint: - active: filters infractions that are active (true), expired (false), or either (not present/any) - expand: expands the result data with the information about the users (slower) + - dangling: filters infractions that are active, or inactive infractions that have not been closed manually. + - search: filters the "reason" field to match the given RE2 query. Infraction Schema: This schema is used when an infraction's data is returned. @@ -32,22 +34,22 @@ Endpoints: GET /bot/infractions Gets a list of all infractions, regardless of type or user. - Parameters: "active", "expand". + Parameters: "active", "expand", "dangling", "search". This endpoint returns an array of infraction objects. GET /bot/infractions/user/ Gets a list of all infractions for a user. - Parameters: "active", "expand". + Parameters: "active", "expand", "search". This endpoint returns an array of infraction objects. GET /bot/infractions/type/ Gets a list of all infractions of the given type (ban, mute, etc.) - Parameters: "active", "expand". + Parameters: "active", "expand", "search". This endpoint returns an array of infraction objects. GET /bot/infractions/user// Gets a list of all infractions of the given type for a user. - Parameters: "active", "expand". + Parameters: "active", "expand", "search". This endpoint returns an array of infraction objects. GET /bot/infractions/user///current @@ -82,7 +84,8 @@ Endpoints: "duration" (optional str): if provided, updates the expiration of the infraction to the time of UPDATING plus the duration. If set to null, the expiration is also set to null (may imply permanence). "active" (optional bool): if provided, activates or deactivates the infraction. This does not do anything - if the infraction isn't duration-based, or if the infraction has already expired. + if the infraction isn't duration-based, or if the infraction has already expired. This marks the infraction + as closed. "expand" (optional bool): whether to expand the infraction user data once the infraction is updated and returned. """ @@ -105,6 +108,8 @@ class InfractionType(NamedTuple): RFC1123_FORMAT = "%a, %d %b %Y %H:%M:%S GMT" +EXCLUDED_FIELDS = "user_id", "actor_id", "closed", "_timed" +INFRACTION_ORDER = rethinkdb.desc("active"), rethinkdb.desc("inserted_at") INFRACTION_TYPES = { "warning": InfractionType(timed_infraction=False), @@ -116,7 +121,9 @@ INFRACTION_TYPES = { GET_SCHEMA = Schema({ Optional("active"): str, - Optional("expand"): str + Optional("expand"): str, + Optional("dangling"): str, + Optional("search"): str }) GET_ACTIVE_SCHEMA = Schema({ @@ -166,8 +173,11 @@ class InfractionsView(APIView, DBMixin): @api_key @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) - def get(self, params=None): - return _infraction_list_filtered(self, params, {}) + def get(self, params: dict = None): + if "dangling" in params: + return _infraction_list_filtered(self, params, {"_timed": True, "closed": False}) + else: + return _infraction_list_filtered(self, params, {}) @api_key @api_params(schema=CREATE_INFRACTION_SCHEMA, validation_type=ValidationTypes.json) @@ -199,7 +209,7 @@ class InfractionsView(APIView, DBMixin): deactivate_infraction_query = \ self.db.query(self.table_name) \ .get(active_infraction["id"]) \ - .update({"active": False}) + .update({"active": False, "closed": True}) if duration_str: try: @@ -227,7 +237,7 @@ class InfractionsView(APIView, DBMixin): query = self.db.query(self.table_name).get(infraction_id) \ .merge(_merge_expand_users(self, expand)) \ .merge(_merge_active_check()) \ - .without("user_id", "actor_id").default(None) + .without(*EXCLUDED_FIELDS).default(None) return jsonify({ "infraction": self.db.run(query) }) @@ -245,6 +255,7 @@ class InfractionsView(APIView, DBMixin): if "active" in data: update_collection["active"] = data["active"] + update_collection["closed"] = not data["active"] if "duration" in data: duration_str = data["duration"] @@ -264,14 +275,15 @@ class InfractionsView(APIView, DBMixin): if not result_update["replaced"]: return jsonify({ - "success": False + "success": False, + "error_message": "Unknown infraction / nothing was changed." }) # return the updated infraction query = self.db.query(self.table_name).get(data["id"]) \ .merge(_merge_expand_users(self, expand)) \ .merge(_merge_active_check()) \ - .without("user_id", "actor_id").default(None) + .without(*EXCLUDED_FIELDS).default(None) infraction = self.db.run(query) return jsonify({ @@ -294,7 +306,7 @@ class InfractionById(APIView, DBMixin): query = self.db.query(self.table_name).get(infraction_id) \ .merge(_merge_expand_users(self, expand)) \ .merge(_merge_active_check()) \ - .without("user_id", "actor_id").default(None) + .without(*EXCLUDED_FIELDS).default(None) return jsonify({ "infraction": self.db.run(query) }) @@ -458,21 +470,34 @@ def _infraction_list_filtered(view, params=None, query_filter=None): query_filter = query_filter or {} active = parse_bool(params.get("active")) expand = parse_bool(params.get("expand"), default=False) + search = params.get("search") if active is not None: query_filter["active"] = active query = _merged_query(view, expand, query_filter) + + if search is not None: + query = query.filter( + lambda row: rethinkdb.branch( + row["reason"].eq(None), + False, + row["reason"].match(search) + ) + ) + + query = query.order_by(*INFRACTION_ORDER) + return jsonify(view.db.run(query.coerce_to("array"))) def _merged_query(view, expand, query_filter): return view.db.query(view.table_name).merge(_merge_active_check()).filter(query_filter) \ - .merge(_merge_expand_users(view, expand)).without("user_id", "actor_id") + .merge(_merge_expand_users(view, expand)).without(*EXCLUDED_FIELDS) def _merge_active_check(): - # Checks if the "active" field has been set to false (manual infraction removal). + # Checks if the "closed" field has been set to true (manual infraction removal). # If not, the "active" field is set to whether the infraction has expired. def _merge(row): return { @@ -480,7 +505,7 @@ def _merge_active_check(): rethinkdb.branch( _is_timed_infraction(row["type"]), rethinkdb.branch( - row["active"].default(True).eq(False), + (row["closed"].default(False).eq(True)) | (row["active"].default(True).eq(False)), False, rethinkdb.branch( row["expires_at"].eq(None), @@ -489,7 +514,9 @@ def _merge_active_check(): ) ), False - ) + ), + "closed": row["closed"].default(False), + "_timed": _is_timed_infraction(row["type"]) } return _merge -- cgit v1.2.3 From 14be9e30deae5714a3bdcd7e0bfe3cddf8fd1844 Mon Sep 17 00:00:00 2001 From: Gareth Coles Date: Fri, 27 Jul 2018 17:10:19 +0100 Subject: Don't remove basic user objects, add API for querying them Also update privacy policy in accordance with this --- pysite/views/api/bot/user.py | 35 +++++++++++++++--- templates/main/about/privacy.html | 77 ++++++++++++++++++++++++++------------- 2 files changed, 82 insertions(+), 30 deletions(-) (limited to 'pysite') diff --git a/pysite/views/api/bot/user.py b/pysite/views/api/bot/user.py index 189dd1f8..c8d769d5 100644 --- a/pysite/views/api/bot/user.py +++ b/pysite/views/api/bot/user.py @@ -1,4 +1,5 @@ import logging +import rethinkdb from flask import jsonify, request from schema import Optional, Schema @@ -18,6 +19,12 @@ SCHEMA = Schema([ } ]) +GET_SCHEMA = Schema([ + { + "user_id": str + } +]) + DELETE_SCHEMA = Schema([ { "user_id": str, @@ -45,6 +52,24 @@ class UserView(APIView, DBMixin): table_name = "users" teams_table = "code_jam_teams" + @api_key + @api_params(schema=GET_SCHEMA, validation_type=ValidationTypes.params) + def get(self, data): + logging.getLogger(__name__).debug(f"Size of request: {len(request.data)} bytes") + + if not data: + return self.error(ErrorCodes.bad_data_format, "No user IDs supplied") + + data = [x["user_id"] for x in data] + + result = self.db.run( + self.db.query(self.table_name) + .filter(lambda document: rethinkdb.expr(data).contains(document["user_id"])), + coerce=list + ) + + return jsonify({"result": result}) # pragma: no cover + @api_key @api_params(schema=SCHEMA, validation_type=ValidationTypes.json) def post(self, data): @@ -72,11 +97,11 @@ class UserView(APIView, DBMixin): def delete(self, data): user_ids = [user["user_id"] for user in data] - changes = self.db.run( - self.db.query(self.table_name) - .get_all(*user_ids) - .delete() - ) + # changes = self.db.run( + # self.db.query(self.table_name) + # .get_all(*user_ids) + # .delete() + # ) oauth_deletions = self.db.run( self.db.query(self.oauth_table_name) diff --git a/templates/main/about/privacy.html b/templates/main/about/privacy.html index b1d778c2..fa4e2aab 100644 --- a/templates/main/about/privacy.html +++ b/templates/main/about/privacy.html @@ -31,49 +31,62 @@

Data collection

+
+

+ Please note that data marked with blurple text below is not + automatically removed. We need to hold onto this information in order to maintain infraction records + and ensure the smooth running of our community. +

+

+ We do not store any data until you have verified yourself in #checkpoint on the server, + and certified that you agree to our rules and privacy policy. If you are leaving the server and would + like us to remove this data as well, please contact a member of staff directly. +

+
+ - - - - + + + + - - + + - - + + - + - - - - + + + + - - - + + + - - + + - + @@ -136,7 +149,7 @@

If you joined the community after the 20th of May, 2018, you will have been greeted with the - #checkpoint channel. In this channel, you must run the self.accept() + #checkpoint channel. In this channel, you must run the !accept command to signify that you accept both our rules and this privacy policy. This will also have been detailed in a message in that channel.

@@ -161,7 +174,7 @@

-
+

Complete data removal

@@ -173,6 +186,12 @@ is leave the Discord server. As much of the data we collect is necessary for running our community, we are unable to offer you community membership with zero data collection.

+

+ Please note that data marked with blurple text in the table above + is not automatically removed. We need to hold onto this information in order to maintain infraction records + and ensure the smooth running of our community. If you are leaving the server and would like us to remove + this data as well, please contact a member of staff directly. +

Once you've left the Discord server, your data is removed automatically. Please note that for the sake of data integrity and moderation purposes, we do not remove your Discord @@ -186,7 +205,7 @@

-
+

Code jam profile removal

@@ -252,6 +271,14 @@

    +
  • +

    July 27th, 2018

    +

    + As we're replacing Rowboat (the bot we use for moderation), we need to hold onto some of + your data - even after you've left the server. This is necessary to ensure the smooth + running and security of our community. +

    +

  • July 3rd, 2018

    -- cgit v1.2.3 From 3f615819febbe7dee7887fe72fb7fd2bdb85def8 Mon Sep 17 00:00:00 2001 From: Gareth Coles Date: Fri, 27 Jul 2018 17:14:49 +0100 Subject: Fix a couple dumb mistakes --- pysite/views/api/bot/user.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'pysite') diff --git a/pysite/views/api/bot/user.py b/pysite/views/api/bot/user.py index c8d769d5..5a096f2c 100644 --- a/pysite/views/api/bot/user.py +++ b/pysite/views/api/bot/user.py @@ -1,6 +1,6 @@ import logging -import rethinkdb +import rethinkdb from flask import jsonify, request from schema import Optional, Schema @@ -97,6 +97,8 @@ class UserView(APIView, DBMixin): def delete(self, data): user_ids = [user["user_id"] for user in data] + changes = {} + # changes = self.db.run( # self.db.query(self.table_name) # .get_all(*user_ids) -- cgit v1.2.3 From 5c90069fef25907c8299b1e50235cc15ff3fa8d3 Mon Sep 17 00:00:00 2001 From: Gareth Coles Date: Fri, 27 Jul 2018 17:20:41 +0100 Subject: Fix unit test --- pysite/views/api/bot/user.py | 2 +- tests/test_api_bot_users.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'pysite') diff --git a/pysite/views/api/bot/user.py b/pysite/views/api/bot/user.py index 5a096f2c..a3a0c7a8 100644 --- a/pysite/views/api/bot/user.py +++ b/pysite/views/api/bot/user.py @@ -68,7 +68,7 @@ class UserView(APIView, DBMixin): coerce=list ) - return jsonify({"result": result}) # pragma: no cover + return jsonify({"data": result}) # pragma: no cover @api_key @api_params(schema=SCHEMA, validation_type=ValidationTypes.json) diff --git a/tests/test_api_bot_users.py b/tests/test_api_bot_users.py index 9ad46071..eda3713e 100644 --- a/tests/test_api_bot_users.py +++ b/tests/test_api_bot_users.py @@ -1,7 +1,7 @@ -import os import json from tests import SiteTest, app + class ApiBotUsersEndpoint(SiteTest): def test_api_user(self): """ Check insert user """ @@ -12,8 +12,8 @@ class ApiBotUsersEndpoint(SiteTest): {'user_id': "1234", 'roles': ["5678"], "username": "test", "discriminator": "0000", "avatar": "http://some/url"} ]) - response = self.client.get('/bot/users', app.config['API_SUBDOMAIN'], headers=app.config['TEST_HEADER']) - self.assertEqual(response.status_code, 405) + response = self.client.get('/bot/users?user_id=1234', app.config['API_SUBDOMAIN'], headers=app.config['TEST_HEADER']) + self.assertTrue("data" in response.json) response = self.client.post('/bot/users', app.config['API_SUBDOMAIN'], headers=app.config['TEST_HEADER'], data=single_data) self.assertTrue("success" in response.json) -- cgit v1.2.3 From 63ebca8e05293ba54e5057a0f5a783ea5b7fdc42 Mon Sep 17 00:00:00 2001 From: Gareth Coles Date: Sat, 28 Jul 2018 18:21:30 +0100 Subject: Fix typo in infractions table definition --- pysite/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pysite') diff --git a/pysite/tables.py b/pysite/tables.py index 838d69f9..6eeda60e 100644 --- a/pysite/tables.py +++ b/pysite/tables.py @@ -266,7 +266,7 @@ TABLES = { "user_id", # str "actor_id", # str "reason", # str - "type" # str + "type", # str "inserted_at", # datetime "expires_at", # datetime "closed", # bool -- cgit v1.2.3

What we collectWhenWhat it's used forWho can access itWhat we collectWhen it's collectedWhat it's used forWho can access it
Discord user IDself.accept() run on DiscordDiscord user ID!accept run on Discord Statistics, data association (infractions, code jam applications, etc) Administrative staff
Discord username and discriminatorself.accept() run on DiscordDiscord username and discriminator!accept run on Discord Display purposes (alongside ID in staff areas, public profiles)Public, for code jam team listings and winner infoPublic (for code jam team listings and winner info) and staff areas
Discord avatar URLsself.accept() run on DiscordDisplay purposes (public profiles)Public, for code jam team listings and winner infoDiscord avatar URLs!accept run on DiscordDisplay purposes (alongside ID in staff areas, public profiles)Public (for code jam team listings and winner info) and staff areas
Assigned roles on Discordself.accept() run on DiscordAccess control for the siteAssigned roles on Discord!accept run on DiscordAccess control for the site, infractions, role restoration after kicks Administrative staff
Messages sent on Discordself.accept() run on DiscordMessages sent on Discord!accept run on Discord - Stored in memory by the bot for processing temporarily, no message content reaches - the database unless you're using a bot command that interfaces with the site - May be - temporarily written to a log file for debugging purposes + Stored in memory by the bot for processing temporarily, may also end up in + staff-only logging channels for the purposes of accountability and infraction + management N/AAdministrative staff