diff options
| -rw-r--r-- | .coveragerc | 9 | ||||
| -rw-r--r-- | .travis.yml | 31 | ||||
| -rw-r--r-- | Pipfile | 4 | ||||
| -rw-r--r-- | Pipfile.lock | 109 | ||||
| -rw-r--r-- | binaries/nsjail | bin | 0 -> 678704 bytes | |||
| -rw-r--r-- | docker-compose.yml | 1 | ||||
| -rw-r--r-- | docker/Dockerfile | 8 | ||||
| -rw-r--r-- | snekbox.py | 160 | ||||
| -rw-r--r-- | tests/__init__.py | 1 | ||||
| -rw-r--r-- | tests/test_snekbox.py | 7 |
10 files changed, 250 insertions, 80 deletions
diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..6e65167 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +[run] +omit = .venv/*, + tests/*, + snekweb.py + +[report] +exclude_lines = return jsonify, + raise RuntimeError, + return diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..075f9c7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +language: python +python: + - "3.6" + +branches: + only: + - "master" + +#sudo: required + +#services: +# - docker + +env: + global: + - PIPENV_VENV_IN_PROJECT=1 + - PIPENV_IGNORE_VIRTUALENVS=1 + +install: + - pip install pipenv + - pipenv sync --dev --three +script: + - pipenv run lint + - pipenv run test +#after_success: +# - bash scripts/deploy.sh + +cache: pip + +notifications: + email: false @@ -14,12 +14,16 @@ gevent = "==1.2.2" gevent-websocket = "*" gunicorn = "*" "flake8" = "*" +pytest = "*" +pytest-cov = "*" +pytest-dependency = "*" [requires] python_version = "3.6" [scripts] lint = "flake8" +test = "py.test tests --cov . --cov-report term-missing -v" snekbox = "python snekbox.py" snekweb = "gunicorn -w 2 -b 0.0.0.0:5000 --log-level debug -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker snekweb:app" buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." diff --git a/Pipfile.lock b/Pipfile.lock index 7bb08b6..70f7e24 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "293e67cbad98e54a08d90526cb01eb524040819f89b942beff3425d7442d0ba9" + "sha256": "56429edc3ce0dd8b29d5c50fa05e864d0a26ba9c3cc844945b116f8c52310801" }, "pipfile-spec": 6, "requires": { @@ -26,6 +26,20 @@ } }, "develop": { + "atomicwrites": { + "hashes": [ + "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585", + "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6" + ], + "version": "==1.1.5" + }, + "attrs": { + "hashes": [ + "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", + "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + ], + "version": "==18.1.0" + }, "certifi": { "hashes": [ "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", @@ -47,6 +61,47 @@ ], "version": "==6.7" }, + "coverage": { + "hashes": [ + "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", + "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed", + "sha256:104ab3934abaf5be871a583541e8829d6c19ce7bde2923b2751e0d3ca44db60a", + "sha256:15b111b6a0f46ee1a485414a52a7ad1d703bdf984e9ed3c288a4414d3871dcbd", + "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640", + "sha256:1c383d2ef13ade2acc636556fd544dba6e14fa30755f26812f54300e401f98f2", + "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162", + "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508", + "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249", + "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694", + "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a", + "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287", + "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1", + "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000", + "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1", + "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e", + "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5", + "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062", + "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba", + "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc", + "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc", + "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99", + "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653", + "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c", + "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558", + "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f", + "sha256:9e112fcbe0148a6fa4f0a02e8d58e94470fc6cb82a5481618fea901699bf34c4", + "sha256:ac4fef68da01116a5c117eba4dd46f2e06847a497de5ed1d64bb99a5fda1ef91", + "sha256:b8815995e050764c8610dbc82641807d196927c3dbed207f0a079833ffcf588d", + "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9", + "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd", + "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d", + "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", + "sha256:e4d96c07229f58cb686120f168276e434660e4358cc9cf3b0464210b04913e77", + "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80", + "sha256:f8a923a85cb099422ad5a2e345fe877bbc89a8a8b23235824a93488150e45f6e" + ], + "version": "==4.5.1" + }, "docker": { "hashes": [ "sha256:43b45b92bed372161a5d4f3c7137e16b30d93845e99a00bc727938e52850694e", @@ -192,6 +247,29 @@ ], "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8", + "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3", + "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0" + ], + "version": "==4.2.0" + }, + "pluggy": { + "hashes": [ + "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff", + "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c", + "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5" + ], + "version": "==0.6.0" + }, + "py": { + "hashes": [ + "sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881", + "sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a" + ], + "version": "==1.5.3" + }, "pycodestyle": { "hashes": [ "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", @@ -206,6 +284,29 @@ ], "version": "==1.6.0" }, + "pytest": { + "hashes": [ + "sha256:39555d023af3200d004d09e51b4dd9fdd828baa863cded3fd6ba2f29f757ae2d", + "sha256:c76e93f3145a44812955e8d46cdd302d8a45fbfc7bf22be24fe231f9d8d8853a" + ], + "index": "pypi", + "version": "==3.6.0" + }, + "pytest-cov": { + "hashes": [ + "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d", + "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec" + ], + "index": "pypi", + "version": "==2.5.1" + }, + "pytest-dependency": { + "hashes": [ + "sha256:895e5b9444fc57a84ff0d5e3fcb7ad8cb7081e6049eaad5c3b9c3419dd0c91d3" + ], + "index": "pypi", + "version": "==0.3.2" + }, "requests": { "hashes": [ "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", @@ -229,10 +330,10 @@ }, "websocket-client": { "hashes": [ - "sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6", - "sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149" + "sha256:18f1170e6a1b5463986739d9fd45c4308b0d025c1b2f9b88788d8f69e8a5eb4a", + "sha256:db70953ae4a064698b27ae56dcad84d0ee68b7b43cb40940f537738f38f510c1" ], - "version": "==0.47.0" + "version": "==0.48.0" }, "werkzeug": { "hashes": [ diff --git a/binaries/nsjail b/binaries/nsjail Binary files differnew file mode 100644 index 0000000..9af91fc --- /dev/null +++ b/binaries/nsjail diff --git a/docker-compose.yml b/docker-compose.yml index db75de3..b3318c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: RABBITMQ_DEFAULT_PASS: rabbits pdsnekbox: + privileged: true hostname: "pdsnekbox" image: pythondiscord/snekbox:latest networks: diff --git a/docker/Dockerfile b/docker/Dockerfile index cb25d34..52e6b25 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,7 @@ FROM python:3.6-alpine3.7 -RUN apk add --update tini +RUN apk add --no-cache libstdc++ protobuf RUN apk add --update build-base -RUN addgroup -g 1000 -S snek && adduser -u 1000 -S snek -G snek ENV PIPENV_VENV_IN_PROJECT=1 ENV PIPENV_IGNORE_VIRTUALENVS=1 @@ -20,8 +19,7 @@ WORKDIR /snekbox RUN pipenv sync -RUN chown -R snek:snek /snekbox -USER snek +RUN cp binaries/nsjail /usr/sbin/nsjail +RUN chmod +x /usr/sbin/nsjail -ENTRYPOINT ["/sbin/tini", "--"] CMD ["pipenv", "run", "snekbox"] @@ -1,83 +1,101 @@ -import sys -import io import json import multiprocessing +import subprocess import threading import time from logs import log from rmq import Rmq -rmq = Rmq() - -def execute(body): - msg = body.decode('utf-8') - log.info(f"incoming: {msg}") - - failed = False - - old_stdout = sys.stdout - old_stderr = sys.stderr - redirected_output = sys.stdout = io.StringIO() - redirected_error = sys.stderr = io.StringIO() - snek_msg = json.loads(msg) - snekid = snek_msg['snekid'] - snekcode = snek_msg['message'].strip() - - try: - exec(snekcode) - - except Exception as e: - failed = str(e) - - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - - if failed: - result = failed.strip() - log.debug(f"this was captured via exception: {result}") - - result_err = redirected_error.getvalue().strip() - result_ok = redirected_output.getvalue().strip() - - if result_err: - log.debug(f"this was captured via stderr: {result_err}") - result = result_err - if result_ok: - result = result_ok - - log.info(f"outgoing: {result}") - - rmq.publish(result, - queue=snekid, - routingkey=snekid, - exchange=snekid) - exit(0) - -def stopwatch(process): - log.debug(f"10 second timer started for process {process.pid}") - for _ in range(10): - time.sleep(1) - if not process.is_alive(): - log.debug(f"Clean exit on process {process.pid}") - exit(0) - - process.terminate() - log.debug(f"Terminated process {process.pid} forcefully") - -def message_handler(ch, method, properties, body, thread_ws=None): - p = multiprocessing.Process(target=execute, args=(body,)) - p.daemon = True - p.start() - - t = threading.Thread(target=stopwatch, args=(p,)) - t.daemon = True - t.start() - - ch.basic_ack(delivery_tag=method.delivery_tag) +class Snekbox(object): + env = { + 'PATH': '/snekbox/.venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + 'LANG': 'en_US.UTF-8', + 'PYTHON_VERSION': '3.6.5', + 'PYTHON_PIP_VERSION': '10.0.1', + 'PYTHONDONTWRITEBYTECODE': '1', + } + + def python3(self, cmd): + args = ["nsjail", "-Mo", + "--rlimit_as", "700", + "--chroot", "/", + "-E", "LANG=en_US.UTF-8", + "-R/usr", "-R/lib", "-R/lib64", + "--user", "nobody", + "--group", "nogroup", + "--time_limit", "2", + "--disable_proc", + "--iface_no_lo", + "--quiet", "--", "/usr/local/bin/python3.6", "-ISq", "-c", cmd] + + proc = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + universal_newlines=True) + + stdout, stderr = proc.communicate() + if proc.returncode == 0: + output = stdout + elif proc.returncode == 1: + try: + output = stderr.split('\n')[-2] + except IndexError: + output = '' + elif proc.returncode == 109: + output = 'timed out or memory limit exceeded' + else: + output = 'unknown error' + return output + + def execute(self, body): + msg = body.decode('utf-8') + log.info(f"incoming: {msg}") + result = "" + snek_msg = json.loads(msg) + snekid = snek_msg['snekid'] + snekcode = snek_msg['message'].strip() + + result = self.python3(snekcode) + + log.info(f"outgoing: {result}") + + rmq.publish(result, + queue=snekid, + routingkey=snekid, + exchange=snekid) + exit(0) + + def stopwatch(self, process): + log.debug(f"10 second timer started for process {process.pid}") + for _ in range(10): + time.sleep(1) + if not process.is_alive(): + log.debug(f"Clean exit on process {process.pid}") + exit(0) + + process.terminate() + log.debug(f"Terminated process {process.pid} forcefully") + + def message_handler(self, ch, method, properties, body, thread_ws=None): + p = multiprocessing.Process(target=self.execute, args=(body,)) + p.daemon = True + p.start() + t = threading.Thread(target=self.stopwatch, args=(p,)) + t.daemon = True + t.start() + + ch.basic_ack(delivery_tag=method.delivery_tag) if __name__ == '__main__': - rmq.consume(callback=message_handler) + try: + rmq = Rmq() + snkbx = Snekbox() + rmq.consume(callback=snkbx.message_handler) + except KeyboardInterrupt: + print("Exited") + exit(0) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/test_snekbox.py b/tests/test_snekbox.py new file mode 100644 index 0000000..5c4d1c7 --- /dev/null +++ b/tests/test_snekbox.py @@ -0,0 +1,7 @@ +import unittest +import pytest + +from snekbox import Snekbox +snek = Snekbox() + +# Write some tests at some point |