diff options
| author | 2022-11-16 23:46:28 -0500 | |
|---|---|---|
| committer | 2022-11-16 23:46:28 -0500 | |
| commit | b92f1b4cebe932dd6ee4f7513ab5dd637e43de2f (patch) | |
| tree | 42bf865ef706f63535ffb51c466b478e5fc325d5 | |
| parent | Add example for post response (diff) | |
Refactored max size handling
| -rw-r--r-- | snekbox/memfs.py | 67 | ||||
| -rw-r--r-- | snekbox/nsjail.py | 29 | ||||
| -rw-r--r-- | snekbox/snekio.py | 21 | 
3 files changed, 52 insertions, 65 deletions
| diff --git a/snekbox/memfs.py b/snekbox/memfs.py index 607aa86..035efed 100644 --- a/snekbox/memfs.py +++ b/snekbox/memfs.py @@ -16,13 +16,12 @@ from snekbox.snekio import FileAttachment  log = logging.getLogger(__name__) -  NAMESPACE_DIR = Path("/memfs")  NAMESPACE_DIR.mkdir(exist_ok=True)  NAMESPACE_DIR.chmod(0o711)  # Execute only access for other users -def mount_tmpfs(name: str) -> Path: +def mount_tmpfs(name: str, size: int | str) -> Path:      """Create and mount a tmpfs directory."""      tmp = NAMESPACE_DIR / name      tmp.mkdir() @@ -34,7 +33,7 @@ def mount_tmpfs(name: str) -> Path:              "-t",              "tmpfs",              "-o", -            f"size={MemFSOptions.MEMFS_SIZE_STR}", +            f"size={size}",              "tmpfs",              str(tmp),          ] @@ -49,29 +48,20 @@ def unmount_tmpfs(name: str) -> None:      rmtree(tmp, ignore_errors=True) -class MemFSOptions: -    """Options for memory file system.""" - -    # Size of the memory filesystem (per instance) -    MEMFS_SIZE = 48 * 1024 * 1024 -    MEMFS_SIZE_STR = "48M" -    # Maximum number of files attachments will be scanned for -    MAX_FILES = 6 -    # Maximum size of a file attachment (8 MiB) -    # 8 MB is also the discord bot upload limit -    MAX_FILE_SIZE = 8 * 1024 * 1024 -    # Size of /dev/shm (16 MiB) -    SHM_SIZE = 16 * 1024 * 1024 - - -class MemoryTempDir: +class MemFS:      """A temporary directory using tmpfs."""      assignment_lock = BoundedSemaphore(1)      assigned_names: set[str] = set()  # Pool of tempdir names in use -    def __init__(self) -> None: +    def __init__(self, instance_size: int) -> None: +        """ +        Create a temporary directory using tmpfs. + +        size: Size limit of each tmpfs instance in bytes +        """          self.path: Path | None = None +        self.instance_size = instance_size      @property      def name(self) -> str | None: @@ -88,24 +78,20 @@ class MemoryTempDir:          """Path to /dev/shm."""          return Path(self.path, "dev", "shm") if self.path else None -    def __enter__(self) -> MemoryTempDir: +    def mkdir(self, path: str, chmod: int = 0o777) -> Path: +        """Create a directory in the tempdir.""" +        f = Path(self.path, path) +        f.mkdir(parents=True, exist_ok=True) +        f.chmod(chmod) +        return f + +    def __enter__(self) -> MemFS:          # Generates a uuid tempdir          with self.assignment_lock:              for _ in range(10):                  name = str(uuid4())                  if name not in self.assigned_names: -                    self.path = mount_tmpfs(name) - -                    # Create a home folder -                    home = self.path / "home" -                    home.mkdir() -                    home.chmod(0o777) - -                    # Create a /dev/shm folder -                    shm = self.path / "dev" / "shm" -                    shm.mkdir(parents=True) -                    shm.chmod(0o777) - +                    self.path = mount_tmpfs(name, self.instance_size)                      self.assigned_names.add(name)                      return self              else: @@ -122,24 +108,27 @@ class MemoryTempDir:      @contextmanager      def allow_write(self) -> None:          """Temporarily allow writes to the root tempdir.""" +        backup = self.path.stat().st_mode          self.path.chmod(0o777)          yield -        self.path.chmod(0o711) +        self.path.chmod(backup) -    def attachments(self) -> Generator[FileAttachment, None, None]: +    def attachments( +        self, max_count: int, max_size: int | None = None +    ) -> Generator[FileAttachment, None, None]:          """Return a list of attachments in the tempdir.""" -        # Look for any file starting with `output`          count = 0 +        # Look for any file starting with `output`          for file in self.home.glob("output*"): -            if count >= MemFSOptions.MAX_FILES: +            if count > max_count:                  log.warning("Maximum number of attachments reached, skipping remaining files")                  break              if file.is_file():                  count += 1 -                yield FileAttachment.from_path(file, MemFSOptions.MAX_FILE_SIZE) +                yield FileAttachment.from_path(file, max_size)      def cleanup(self) -> None: -        """Remove files in temp dir, releases name.""" +        """Unmounts tmpfs, releases name."""          if self.path is None:              return          # Remove the path folder diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index 47a5860..8b3f6cd 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -11,7 +11,7 @@ from google.protobuf import text_format  # noinspection PyProtectedMember  from snekbox import DEBUG, utils  from snekbox.config_pb2 import NsJailConfig -from snekbox.memfs import MemFSOptions, MemoryTempDir +from snekbox.memfs import MemFS  __all__ = ("NsJail",) @@ -39,11 +39,17 @@ class NsJail:          config_path: str = "./config/snekbox.cfg",          max_output_size: int = 1_000_000,          read_chunk_size: int = 10_000, +        memfs_instance_size: int = 48 * 1024 * 1024, +        max_attachments: int = 100, +        max_attachment_size: int | None = None,      ):          self.nsjail_path = nsjail_path          self.config_path = config_path          self.max_output_size = max_output_size          self.read_chunk_size = read_chunk_size +        self.memfs_instance_size = memfs_instance_size +        self.max_attachments = max_attachments +        self.max_attachment_size = max_attachment_size          self.config = self._read_config(config_path)          self.cgroup_version = utils.cgroup.init(self.config) @@ -133,7 +139,11 @@ class NsJail:          return "".join(output)      def python3( -        self, code: str, *, nsjail_args: Iterable[str] = (), py_args: Iterable[str] = ("",) +        self, +        code: str, +        *, +        nsjail_args: Iterable[str] = (), +        py_args: Iterable[str] = ("",),      ) -> EvalResult:          """          Execute Python 3 code in an isolated environment and return the completed process. @@ -156,16 +166,16 @@ class NsJail:                  *nsjail_args,              ) -        with NamedTemporaryFile() as nsj_log, MemoryTempDir() as temp_dir: +        with NamedTemporaryFile() as nsj_log, MemFS(self.memfs_instance_size) as fs:              # Add the temp dir to be mounted as cwd              nsjail_args = (                  # Mount a tmpfs at /dev/shm to support multiprocessing                  "--mount",                  # src:dst:fs_type:options -                f"{temp_dir.shm}:/dev/shm:tmpfs:size={MemFSOptions.SHM_SIZE}", +                f"{fs.shm}:/dev/shm:tmpfs:size={fs.instance_size}",                  # Mount `home` in R/W mode                  "--bindmount", -                f"{temp_dir.home}:home", +                f"{fs.home}:home",                  # Set cwd to temp dir                  "--cwd",                  "home", @@ -198,8 +208,8 @@ class NsJail:                  log.info(f"args: {args}")                  # Write the code to a file                  if not c_mode: -                    with temp_dir.allow_write(): -                        code_path = temp_dir.home / "main.py" +                    with fs.allow_write(): +                        code_path = fs.home / "main.py"                          code_path.write_text(code)                      log.info(f"Created code file at [{code_path!r}].")              else: @@ -230,7 +240,10 @@ class NsJail:              # Parse attachments              try:                  # Sort attachments by name lexically -                attachments = sorted(temp_dir.attachments(), key=lambda a: a.name) +                attachments = sorted( +                    fs.attachments(self.max_attachments, self.max_attachment_size), +                    key=lambda a: a.name, +                )                  log.info(f"Found {len(attachments)} attachments.")              except AttachmentError as err:                  log.warning(f"Failed to parse attachments: {err}") diff --git a/snekbox/snekio.py b/snekbox/snekio.py index 12645cb..eac6438 100644 --- a/snekbox/snekio.py +++ b/snekbox/snekio.py @@ -33,36 +33,21 @@ class FileAttachment:      content: bytes      @classmethod -    def from_path(cls, file: Path, max_size: int) -> FileAttachment: +    def from_path(cls, file: Path, max_size: int | None = None) -> FileAttachment:          """Create an attachment from a path."""          size = file.stat().st_size -        if size > max_size: +        if max_size is not None and size > max_size:              raise AttachmentError(                  f"File {file.name} too large: {sizeof_fmt(size)} "                  f"exceeds the limit of {sizeof_fmt(max_size)}"              ) - -        with file.open("rb") as f: -            content = f.read(max_size + 1) -            size = len(content) -            if len(content) > max_size: -                raise AttachmentError( -                    f"File {file.name} too large: {sizeof_fmt(len(content))} " -                    f"exceeds the limit of {sizeof_fmt(max_size)}" -                ) -        return cls(file.name, content) +        return cls(file.name, file.read_bytes())      @property      def mime(self) -> str:          """MIME type of the attachment."""          return mimetypes.guess_type(self.name)[0] -    def is_supported(self) -> bool: -        """Return whether the attachment is supported.""" -        if self.mime.startswith("text/"): -            return True -        return self.mime in SUPPORTED_MIME_TYPES -      def to_dict(self) -> dict[str, str]:          """Convert the attachment to a dict."""          cmp = zlib.compress(self.content) | 
