aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bot/exts/filtering/filtering.py9
-rw-r--r--bot/exts/moderation/dm_relay.py9
-rw-r--r--bot/exts/moderation/metabase.py12
-rw-r--r--bot/exts/utils/internal.py12
-rw-r--r--bot/exts/utils/snekbox/_cog.py14
-rw-r--r--bot/utils/__init__.py4
-rw-r--r--bot/utils/services.py83
-rw-r--r--tests/bot/exts/utils/snekbox/test_snekbox.py13
-rw-r--r--tests/bot/utils/test_services.py90
9 files changed, 40 insertions, 206 deletions
diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py
index d69bd9644..629f9471e 100644
--- a/bot/exts/filtering/filtering.py
+++ b/bot/exts/filtering/filtering.py
@@ -18,6 +18,7 @@ from discord.ext import commands, tasks
from discord.ext.commands import BadArgument, Cog, Context, command, has_any_role
from pydis_core.site_api import ResponseCodeError
from pydis_core.utils import scheduling
+from pydis_core.utils.paste_service import PasteTooLongError, PasteUploadError, send_to_paste_service
import bot
import bot.exts.filtering._ui.filter as filters_ui
@@ -47,7 +48,6 @@ from bot.pagination import LinePaginator
from bot.utils.channel import is_mod_channel
from bot.utils.lock import lock_arg
from bot.utils.message_cache import MessageCache
-from bot.utils.services import PasteTooLongError, PasteUploadError, send_to_paste_service
log = get_logger(__name__)
@@ -1452,7 +1452,12 @@ class Filtering(Cog):
raise
report = discord.utils.remove_markdown(report)
try:
- paste_resp = await send_to_paste_service(report, extension="txt")
+ resp = await send_to_paste_service(
+ contents=report,
+ http_session=self.bot.http_session,
+ lexer="text",
+ )
+ paste_resp = resp["link"]
except (ValueError, PasteTooLongError, PasteUploadError):
paste_resp = ":warning: Failed to upload report to paste service"
file_buffer = io.StringIO(report)
diff --git a/bot/exts/moderation/dm_relay.py b/bot/exts/moderation/dm_relay.py
index bf0b96a58..5bec3b10f 100644
--- a/bot/exts/moderation/dm_relay.py
+++ b/bot/exts/moderation/dm_relay.py
@@ -1,11 +1,11 @@
import discord
from discord.ext.commands import Cog, Context, command, has_any_role
+from pydis_core.utils.paste_service import PasteTooLongError, PasteUploadError, send_to_paste_service
from bot.bot import Bot
from bot.constants import Emojis, MODERATION_ROLES
from bot.log import get_logger
from bot.utils.channel import is_mod_channel
-from bot.utils.services import PasteTooLongError, PasteUploadError, send_to_paste_service
log = get_logger(__name__)
@@ -54,7 +54,12 @@ class DMRelay(Cog):
f"Channel ID: {user.dm_channel.id}\n\n"
)
try:
- message = await send_to_paste_service(metadata + output, extension="txt")
+ resp = await send_to_paste_service(
+ contents=metadata + output,
+ lexer="text",
+ http_session=self.bot.http_session,
+ )
+ message = resp["link"]
except PasteTooLongError:
message = f"{Emojis.cross_mark} Too long to upload to paste service."
except PasteUploadError:
diff --git a/bot/exts/moderation/metabase.py b/bot/exts/moderation/metabase.py
index f5483325f..1c0a2c191 100644
--- a/bot/exts/moderation/metabase.py
+++ b/bot/exts/moderation/metabase.py
@@ -9,14 +9,13 @@ from aiohttp.client_exceptions import ClientResponseError
from arrow import Arrow
from async_rediscache import RedisCache
from discord.ext.commands import Cog, Context, group, has_any_role
+from pydis_core.utils.paste_service import PasteTooLongError, PasteUploadError, send_to_paste_service
from pydis_core.utils.scheduling import Scheduler
from bot.bot import Bot
from bot.constants import Metabase as MetabaseConfig, Roles
from bot.log import get_logger
-from bot.utils import send_to_paste_service
from bot.utils.channel import is_mod_channel
-from bot.utils.services import PasteTooLongError, PasteUploadError
log = get_logger(__name__)
@@ -129,6 +128,7 @@ class Metabase(Cog):
async with self.bot.http_session.post(url, headers=self.headers, raise_for_status=True) as resp:
if extension == "csv":
+ extension = "text" # paste site doesn't support csv as a lexer
out = await resp.text(encoding="utf-8")
# Save the output for use with int e
self.exports[question_id] = list(csv.DictReader(StringIO(out)))
@@ -142,13 +142,17 @@ class Metabase(Cog):
out = json.dumps(out, indent=4, sort_keys=True)
try:
- paste_link = await send_to_paste_service(out, extension=extension)
+ resp = await send_to_paste_service(
+ contents=out,
+ lexer=extension,
+ http_session=self.bot.http_session,
+ )
except PasteTooLongError:
message = f":x: {ctx.author.mention} Too long to upload to paste service."
except PasteUploadError:
message = f":x: {ctx.author.mention} Failed to upload to paste service."
else:
- message = f":+1: {ctx.author.mention} Here's your link: {paste_link}"
+ message = f":+1: {ctx.author.mention} Here's your link: {resp['link']}"
await ctx.send(
f"{message}\nYou can also access this data within internal eval by doing: "
diff --git a/bot/exts/utils/internal.py b/bot/exts/utils/internal.py
index cdaa9a61b..58140b3c0 100644
--- a/bot/exts/utils/internal.py
+++ b/bot/exts/utils/internal.py
@@ -11,12 +11,12 @@ from typing import Any
import arrow
import discord
from discord.ext.commands import Cog, Context, group, has_any_role, is_owner
+from pydis_core.utils.paste_service import PasteTooLongError, PasteUploadError, send_to_paste_service
from bot.bot import Bot
from bot.constants import DEBUG_MODE, Roles
from bot.log import get_logger
-from bot.utils import find_nth_occurrence, send_to_paste_service
-from bot.utils.services import PasteTooLongError, PasteUploadError
+from bot.utils import find_nth_occurrence
log = get_logger(__name__)
@@ -196,13 +196,17 @@ async def func(): # (None,) -> Any
if len(out) > truncate_index:
try:
- paste_link = await send_to_paste_service(out, extension="py")
+ resp = await send_to_paste_service(
+ contents=out,
+ lexer="python",
+ http_session=self.bot.http_session,
+ )
except PasteTooLongError:
paste_text = "too long to upload to paste service."
except PasteUploadError:
paste_text = "failed to upload contents to paste service."
else:
- paste_text = f"full contents at {paste_link}"
+ paste_text = f"full contents at {resp['link']}"
await ctx.send(
f"```py\n{out[:truncate_index]}\n```"
diff --git a/bot/exts/utils/snekbox/_cog.py b/bot/exts/utils/snekbox/_cog.py
index 9d0ced4ef..f07bcb7df 100644
--- a/bot/exts/utils/snekbox/_cog.py
+++ b/bot/exts/utils/snekbox/_cog.py
@@ -11,6 +11,7 @@ from typing import Literal, NamedTuple, TYPE_CHECKING
from discord import AllowedMentions, HTTPException, Interaction, Message, NotFound, Reaction, User, enums, ui
from discord.ext.commands import Cog, Command, Context, Converter, command, guild_only
from pydis_core.utils import interactions
+from pydis_core.utils.paste_service import PasteTooLongError, PasteUploadError, send_to_paste_service
from pydis_core.utils.regex import FORMATTED_CODE_REGEX, RAW_CODE_REGEX
from bot.bot import Bot
@@ -21,9 +22,7 @@ from bot.exts.help_channels._channel import is_help_forum_post
from bot.exts.utils.snekbox._eval import EvalJob, EvalResult
from bot.exts.utils.snekbox._io import FileAttachment
from bot.log import get_logger
-from bot.utils import send_to_paste_service
from bot.utils.lock import LockedResourceError, lock_arg
-from bot.utils.services import PasteTooLongError, PasteUploadError
if TYPE_CHECKING:
from bot.exts.filtering.filtering import Filtering
@@ -74,7 +73,6 @@ if not hasattr(sys, "_setup_finished"):
{setup}
"""
-MAX_PASTE_LENGTH = 10_000
# Max to display in a codeblock before sending to a paste service
# This also applies to text files
MAX_OUTPUT_BLOCK_LINES = 10
@@ -205,13 +203,17 @@ class Snekbox(Cog):
async with self.bot.http_session.post(URLs.snekbox_eval_api, json=data, raise_for_status=True) as resp:
return EvalResult.from_dict(await resp.json())
- @staticmethod
- async def upload_output(output: str) -> str | None:
+ async def upload_output(self, output: str) -> str | None:
"""Upload the job's output to a paste service and return a URL to it if successful."""
log.trace("Uploading full output to paste service...")
try:
- return await send_to_paste_service(output, extension="txt", max_length=MAX_PASTE_LENGTH)
+ paste_link = await send_to_paste_service(
+ contents=output,
+ lexer="text",
+ http_session=self.bot.http_session,
+ )
+ return paste_link["link"]
except PasteTooLongError:
return "too long to upload"
except PasteUploadError:
diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py
index f576b7f11..f803add1c 100644
--- a/bot/utils/__init__.py
+++ b/bot/utils/__init__.py
@@ -1,12 +1,8 @@
from bot.utils.helpers import CogABCMeta, find_nth_occurrence, has_lines, pad_base64
-from bot.utils.services import PasteTooLongError, PasteUploadError, send_to_paste_service
__all__ = [
"CogABCMeta",
"find_nth_occurrence",
"has_lines",
"pad_base64",
- "send_to_paste_service",
- "PasteUploadError",
- "PasteTooLongError",
]
diff --git a/bot/utils/services.py b/bot/utils/services.py
deleted file mode 100644
index a25548510..000000000
--- a/bot/utils/services.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from aiohttp import ClientConnectorError
-
-import bot
-from bot.constants import URLs
-from bot.log import get_logger
-
-log = get_logger(__name__)
-
-FAILED_REQUEST_ATTEMPTS = 3
-MAX_PASTE_LENGTH = 100_000
-
-
-class PasteUploadError(Exception):
- """Raised when an error is encountered uploading to the paste service."""
-
-
-class PasteTooLongError(Exception):
- """Raised when content is too large to upload to the paste service."""
-
-
-async def send_to_paste_service(contents: str, *, extension: str = "", max_length: int = MAX_PASTE_LENGTH) -> str:
- """
- Upload `contents` to the paste service.
-
- Add `extension` to the output URL. Use `max_length` to limit the allowed contents length
- to lower than the maximum allowed by the paste service.
-
- Raise `ValueError` if `max_length` is greater than the maximum allowed by the paste service.
- Raise `PasteTooLongError` if `contents` is too long to upload, and `PasteUploadError` if uploading fails.
-
- Return the generated URL with the extension.
- """
- if max_length > MAX_PASTE_LENGTH:
- raise ValueError(f"`max_length` must not be greater than {MAX_PASTE_LENGTH}")
-
- extension = extension and f".{extension}"
-
- contents_size = len(contents.encode())
- if contents_size > max_length:
- log.info("Contents too large to send to paste service.")
- raise PasteTooLongError(f"Contents of size {contents_size} greater than maximum size {max_length}")
-
- log.debug(f"Sending contents of size {contents_size} bytes to paste service.")
- paste_url = URLs.paste_service.format(key="documents")
- for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1):
- try:
- async with bot.instance.http_session.post(paste_url, data=contents) as response:
- response_json = await response.json()
- except ClientConnectorError:
- log.warning(
- f"Failed to connect to paste service at url {paste_url}, "
- f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})."
- )
- continue
- except Exception:
- log.exception(
- f"An unexpected error has occurred during handling of the request, "
- f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})."
- )
- continue
-
- if "message" in response_json:
- log.warning(
- f"Paste service returned error {response_json['message']} with status code {response.status}, "
- f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})."
- )
- continue
- if "key" in response_json:
- log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.")
-
- paste_link = URLs.paste_service.format(key=response_json["key"]) + extension
-
- if extension == ".py":
- return paste_link
-
- return paste_link + "?noredirect"
-
- log.warning(
- f"Got unexpected JSON response from paste service: {response_json}\n"
- f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})."
- )
-
- raise PasteUploadError("Failed to upload contents to paste service")
diff --git a/tests/bot/exts/utils/snekbox/test_snekbox.py b/tests/bot/exts/utils/snekbox/test_snekbox.py
index fa28aade8..6d9b6b1e4 100644
--- a/tests/bot/exts/utils/snekbox/test_snekbox.py
+++ b/tests/bot/exts/utils/snekbox/test_snekbox.py
@@ -5,6 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, call, create_autospec, pat
from discord import AllowedMentions
from discord.ext import commands
+from pydis_core.utils.paste_service import MAX_PASTE_SIZE
from bot import constants
from bot.errors import LockedResourceError
@@ -56,19 +57,9 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):
async def test_upload_output_reject_too_long(self):
"""Reject output longer than MAX_PASTE_LENGTH."""
- result = await self.cog.upload_output("-" * (snekbox._cog.MAX_PASTE_LENGTH + 1))
+ result = await self.cog.upload_output("-" * (MAX_PASTE_SIZE + 1))
self.assertEqual(result, "too long to upload")
- @patch("bot.exts.utils.snekbox._cog.send_to_paste_service")
- async def test_upload_output(self, mock_paste_util):
- """Upload the eval output to the URLs.paste_service.format(key="documents") endpoint."""
- await self.cog.upload_output("Test output.")
- mock_paste_util.assert_called_once_with(
- "Test output.",
- extension="txt",
- max_length=snekbox._cog.MAX_PASTE_LENGTH
- )
-
async def test_codeblock_converter(self):
ctx = MockContext()
cases = (
diff --git a/tests/bot/utils/test_services.py b/tests/bot/utils/test_services.py
deleted file mode 100644
index 3c9e037ce..000000000
--- a/tests/bot/utils/test_services.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import logging
-import unittest
-from unittest.mock import AsyncMock, MagicMock, Mock, patch
-
-from aiohttp import ClientConnectorError
-
-from bot.utils.services import (
- FAILED_REQUEST_ATTEMPTS, MAX_PASTE_LENGTH, PasteTooLongError, PasteUploadError, send_to_paste_service
-)
-from tests.helpers import MockBot
-
-
-class PasteTests(unittest.IsolatedAsyncioTestCase):
- def setUp(self) -> None:
- patcher = patch("bot.instance", new=MockBot())
- self.bot = patcher.start()
- self.addCleanup(patcher.stop)
-
- @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}")
- async def test_url_and_sent_contents(self):
- """Correct url was used and post was called with expected data."""
- response = MagicMock(
- json=AsyncMock(return_value={"key": ""})
- )
- self.bot.http_session.post.return_value.__aenter__.return_value = response
- self.bot.http_session.post.reset_mock()
- await send_to_paste_service("Content")
- self.bot.http_session.post.assert_called_once_with("https://paste_service.com/documents", data="Content")
-
- @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}")
- async def test_paste_returns_correct_url_on_success(self):
- """Url with specified extension is returned on successful requests."""
- key = "paste_key"
- test_cases = (
- (f"https://paste_service.com/{key}.txt?noredirect", "txt"),
- (f"https://paste_service.com/{key}.py", "py"),
- (f"https://paste_service.com/{key}?noredirect", ""),
- )
- response = MagicMock(
- json=AsyncMock(return_value={"key": key})
- )
- self.bot.http_session.post.return_value.__aenter__.return_value = response
-
- for expected_output, extension in test_cases:
- with self.subTest(msg=f"Send contents with extension {extension!r}"):
- self.assertEqual(
- await send_to_paste_service("", extension=extension),
- expected_output
- )
-
- async def test_request_repeated_on_json_errors(self):
- """Json with error message and invalid json are handled as errors and requests repeated."""
- test_cases = ({"message": "error"}, {"unexpected_key": None}, {})
- self.bot.http_session.post.return_value.__aenter__.return_value = response = MagicMock()
- self.bot.http_session.post.reset_mock()
-
- for error_json in test_cases:
- with self.subTest(error_json=error_json):
- response.json = AsyncMock(return_value=error_json)
- with self.assertRaises(PasteUploadError):
- await send_to_paste_service("")
- self.assertEqual(self.bot.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS)
-
- self.bot.http_session.post.reset_mock()
-
- async def test_request_repeated_on_connection_errors(self):
- """Requests are repeated in the case of connection errors."""
- self.bot.http_session.post = MagicMock(side_effect=ClientConnectorError(Mock(), Mock()))
- with self.assertRaises(PasteUploadError):
- await send_to_paste_service("")
- self.assertEqual(self.bot.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS)
-
- async def test_general_error_handled_and_request_repeated(self):
- """All `Exception`s are handled, logged and request repeated."""
- self.bot.http_session.post = MagicMock(side_effect=Exception)
- with self.assertRaises(PasteUploadError):
- await send_to_paste_service("")
- self.assertEqual(self.bot.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS)
- self.assertLogs("bot.utils", logging.ERROR)
-
- async def test_raises_error_on_too_long_input(self):
- """Ensure PasteTooLongError is raised if `contents` is longer than `MAX_PASTE_LENGTH`."""
- contents = "a" * (MAX_PASTE_LENGTH + 1)
- with self.assertRaises(PasteTooLongError):
- await send_to_paste_service(contents)
-
- async def test_raises_on_too_large_max_length(self):
- """Ensure ValueError is raised if `max_length` passed is greater than `MAX_PASTE_LENGTH`."""
- with self.assertRaises(ValueError):
- await send_to_paste_service("Hello World!", max_length=MAX_PASTE_LENGTH + 1)
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-21 20:00:44 +0000