diff options
| -rw-r--r-- | Pipfile | 14 | ||||
| -rw-r--r-- | README.md | 59 | ||||
| -rw-r--r-- | config.py | 35 | ||||
| -rw-r--r-- | docker-compose.yml | 40 | ||||
| -rw-r--r-- | docker/Dockerfile | 3 | ||||
| -rw-r--r-- | docker/Dockerfile.webapp | 25 | ||||
| -rw-r--r-- | rmq.py | 113 | ||||
| -rw-r--r-- | snekbox.py | 58 | ||||
| -rw-r--r-- | snekweb.py | 74 | ||||
| -rw-r--r-- | templates/index.html | 114 | ||||
| -rw-r--r-- | templates/result.html | 9 | ||||
| -rw-r--r-- | tests/test_snekbox.py | 20 |
12 files changed, 76 insertions, 488 deletions
@@ -4,15 +4,12 @@ verify_ssl = true name = "pypi" [packages] -pika = "*" docker = "*" - -[dev-packages] flask = "*" -flask-sockets = "*" -gevent = "==1.2.2" -gevent-websocket = "*" +gevent = "*" gunicorn = "*" + +[dev-packages] pytest = "*" pytest-cov = "*" pytest-dependency = "*" @@ -33,13 +30,10 @@ lint = "flake8" precommit = "pre-commit install" test = "py.test tests --cov . --cov-report term-missing -v" report = "py.test tests --cov . --cov-report=html" -snekbox = "python snekbox.py" -snekweb = "gunicorn -w 2 -b 0.0.0.0:5000 --log-level debug -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker snekweb:app" +snekbox = "gunicorn -w 2 -b 0.0.0.0:8060 snekbox:app" buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." pushbox = "docker push pythondiscord/snekbox:latest" buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ." pushboxbase = "docker push pythondiscord/snekbox-base:latest" buildci = "docker build -t pythondiscord/snekbox-ci:latest -f docker/ci.Dockerfile ." pushci = "docker push pythondiscord/snekbox-ci:latest" -buildweb = "docker build -t pythondiscord/snekboxweb:latest -f docker/Dockerfile.webapp ." -pushweb = "docker push pythondiscord/snekboxweb:latest" @@ -7,21 +7,18 @@ Python sandbox runners for executing code in isolation aka snekbox The user sends a piece of python code to a snekbox, the snekbox executes the code and sends the result back to the users. ``` - +-------------+ +------------+ +-----------+ - input -> | |---------->| |-------->| | >----------+ - | WEBSERVER | | RABBITMQ | | SNEKBOX | execution | -result <- | |<----------| |<--------| | <----------+ - +-------------+ +------------+ +-----------+ - ^ ^ ^ - | | |- Executes python code - | | |- Returns result - | | +----------------------- - | | - | |- Message queues opens on demand and closes automatically - | +--------------------------------------------------------- + +-------------+ +-----------+ + input -> | |---------->| | >----------+ + | HTTP POST | | SNEKBOX | execution | +result <- | |<----------| | <----------+ + +-------------+ +-----------+ + ^ ^ + | |- Executes python code + | |- Returns result + | +----------------------- | - |- Uses websockets for asynchronous connection between webui and webserver - +------------------------------------------------------------------------- + |- HTTP POST Endpoint receives request and returns result + +--------------------------------------------------------- ``` @@ -36,6 +33,9 @@ result <- | |<----------| |<--------| | <------ | docker | 18.03.1-ce | | docker-compose | 1.21.2 | | nsjail | 2.5 | +| flask | 1.0.2 | +| gevent | 1.4 | +| gunicorn | 19.9 | _________________________________________ ## Setup local test @@ -83,38 +83,20 @@ python3.6 -ISq -c "print('test')" ## Development environment -Start a rabbitmq instance and get the container IP +Start the webserver with docker: ```bash -docker-compose up -d pdrmq -docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' rmq -# expected output with default setting: 172.17.0.2 -# If not, change the config.py file to match +docker-compose up -d ``` -rabbitmq webinterface: `http://localhost:15672` - -start the webserver - -```bash -docker-compose up -d pdsnkweb -netstat -plnt -# tcp 0.0.0.0:5000 LISTEN -``` - -`http://localhost:5000` - +Run locally with pipenv: ```bash pipenv run snekbox # for debugging -# or -docker-compose up pdsnk # for running the container ``` - +Visit: `http://localhost:8060` ________________________________________ ## Unit testing and lint -Make sure rabbitmq is running before running tests - ```bash pipenv run lint pipenv run test @@ -126,11 +108,6 @@ ________________________________________ ```bash # Build pipenv run buildbox -pipenv run buildweb - # Push pipenv run pushbox -pipenv run pushweb ``` - - diff --git a/config.py b/config.py deleted file mode 100644 index 5e4f648..0000000 --- a/config.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -import docker -from docker.errors import NotFound - - -def autodiscover(): - """Search for the snekbox container and return its IPv4 address.""" - - container_names = ["rmq", "pdrmq", "snekbox_pdrmq_1"] - - client = docker.from_env() - for name in container_names: - try: - container = client.containers.get(name) - if container.status == "running": - host = list(container.attrs.get('NetworkSettings').get('Networks').values()) - host = host[0]['IPAddress'] - return host - except NotFound: - continue - except Exception: - pass - - return '127.0.0.1' - - -USERNAME = os.environ.get('RMQ_USERNAME', 'guest') -PASSWORD = os.environ.get('RMQ_PASSWORD', 'guest') -HOST = os.environ.get('RMQ_HOST', autodiscover()) -PORT = 5672 -QUEUE = 'input' -EXCHANGE = QUEUE -ROUTING_KEY = QUEUE -EXCHANGE_TYPE = 'direct' diff --git a/docker-compose.yml b/docker-compose.yml index 3aedf14..2b22db4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,7 @@ version: '3' services: - pdrmq: - hostname: "pdrmq" - image: pythondiscord/rmq:latest - expose: - - "15672" - ports: - - "15672:15672" - networks: - - sneknet - environment: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - pdsnk: - privileged: true hostname: "pdsnk" + privileged: true image: pythondiscord/snekbox:latest - networks: - - sneknet - environment: - RMQ_HOST: pdrmq - RMQ_USERNAME: guest - RMQ_PASSWORD: guest - - pdsnkweb: - hostname: "pdsnkweb" - image: pythondiscord/snekboxweb:latest - networks: - - sneknet - ports: - - "5000:5000" - expose: - - "5000" - environment: - RMQ_HOST: pdrmq - RMQ_USERNAME: guest - RMQ_PASSWORD: guest - - -networks: - sneknet: + network_mode: "host" diff --git a/docker/Dockerfile b/docker/Dockerfile index e8fa8a5..b8d5637 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,7 @@ FROM pythondiscord/snekbox-base:latest +RUN apk add --update tini + RUN mkdir -p /snekbox COPY . /snekbox WORKDIR /snekbox @@ -7,4 +9,5 @@ WORKDIR /snekbox RUN pipenv --rm RUN pipenv sync +ENTRYPOINT ["/sbin/tini", "--"] CMD ["pipenv", "run", "snekbox"] diff --git a/docker/Dockerfile.webapp b/docker/Dockerfile.webapp deleted file mode 100644 index 988926d..0000000 --- a/docker/Dockerfile.webapp +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.6.6-alpine3.7 - -RUN apk add --update tini -RUN apk add --update build-base - -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 -ENV PYTHONPATH=/webapp - -RUN pip install pipenv - -RUN mkdir -p /webapp -COPY Pipfile /webapp -COPY Pipfile.lock /webapp -COPY . /webapp -WORKDIR /webapp - -RUN pipenv sync --dev - -EXPOSE 5000 - -ENTRYPOINT ["/sbin/tini", "--"] -CMD ["pipenv", "run", "snekweb"] @@ -1,113 +0,0 @@ -import time -import traceback - -import pika -from pika.exceptions import ConnectionClosed - -from config import EXCHANGE, EXCHANGE_TYPE, HOST, PASSWORD, PORT, QUEUE, ROUTING_KEY, USERNAME -from logs import log - - -class Rmq: - """Rabbit MQ (RMQ) implementation used for communication with the bot.""" - - def __init__(self): - self.credentials = pika.PlainCredentials(USERNAME, PASSWORD) - self.con_params = pika.ConnectionParameters(HOST, PORT, '/', self.credentials) - self.properties = pika.BasicProperties(content_type='text/plain', delivery_mode=1) - - def _declare(self, channel, queue): - channel.queue_declare( - queue=queue, - durable=False, # Do not commit messages to disk - arguments={'x-message-ttl': 5000}, # Delete message automatically after x milliseconds - auto_delete=True) # Delete queue when all connection are closed - - def consume(self, queue=QUEUE, callback=None, thread_ws=None, run_once=False): - """Subscribe to read from a RMQ channel.""" - - while True: - try: - connection = pika.BlockingConnection(self.con_params) - - try: - channel = connection.channel() - self._declare(channel, queue) - channel.basic_qos(prefetch_count=1) - - if not run_once: - channel.basic_consume( - lambda ch, method, properties, body: - callback(ch, method, properties, body, thread_ws=thread_ws), - queue=queue) - - log.info(f"Connected to host: {HOST} port: {PORT} queue: {queue}") - - if thread_ws: - if not thread_ws.closed: - thread_ws.send('{"service": "connected"}') - - if run_once: - return channel.basic_get(queue=queue) - - channel.start_consuming() - - except Exception: - exc = traceback.format_exc() - log.error(exc) - - finally: - connection.close() - - except ConnectionClosed: - if thread_ws: - if not thread_ws.closed: - log.error(f"Connection to {HOST} could not be established") - thread_ws.send('{"service": "disconnected"}') - exit(1) - - log.error(f"Connection lost, reconnecting to {HOST}") - - time.sleep(2) - - def publish(self, message, queue=QUEUE, routingkey=ROUTING_KEY, exchange=EXCHANGE): - """Open a connection to publish (write) to a RMQ channel.""" - - try: - connection = pika.BlockingConnection(self.con_params) - - try: - channel = connection.channel() - - self._declare(channel, queue) - - channel.exchange_declare( - exchange=exchange, - exchange_type=EXCHANGE_TYPE) - - channel.queue_bind( - exchange=exchange, - queue=queue, - routing_key=routingkey) - - result = channel.basic_publish( - exchange=exchange, - routing_key=routingkey, - body=message, - properties=self.properties) - - if result: - return result - - else: - log.error(f"Message '{message}' not delivered") - - except ConnectionClosed: - log.error(f"Could not send message, connection to {HOST} was lost") - exit(1) - - finally: - connection.close() - - except ConnectionClosed: - log.error(f"Could not connect to {HOST}") @@ -1,10 +1,8 @@ -import json -import multiprocessing import os import subprocess import sys -from rmq import Rmq +from flask import Flask, render_template, request, jsonify class Snekbox: @@ -98,46 +96,32 @@ class Snekbox: return output - def execute(self, body): - """ - Handles execution of a raw JSON-formatted RMQ message, contained in ``body``. - The message metadata, including the Python code to be executed, is - extracted from the message body. The code is then executed in the - isolated environment, and the results of the execution published - to RMQ. Once published, the system exits, since the snekboxes - are created and disposed of per-execution. - """ +snekbox = Snekbox() - msg = body.decode('utf-8') - result = '' - snek_msg = json.loads(msg) - snekid = snek_msg['snekid'] - snekcode = snek_msg['message'].strip() +# Load app +app = Flask(__name__) +app.use_reloader = False - result = self.python3(snekcode) +# Logging +log = app.logger - rmq.publish(result, - queue=snekid, - routingkey=snekid, - exchange=snekid) - exit(0) - def message_handler(self, ch, method, properties, body, thread_ws=None): - """Spawns a daemon process that handles RMQ messages.""" [email protected]('/') +def index(): + return render_template('index.html') - p = multiprocessing.Process(target=self.execute, args=(body,)) - p.daemon = True - p.start() - ch.basic_ack(delivery_tag=method.delivery_tag) [email protected]('/result', methods=["POST", "GET"]) +def result(): + if request.method == "POST": + code = request.form["Code"] + output = snekbox.python3(code) + return render_template('result.html', code=code, result=output) -if __name__ == '__main__': - try: - rmq = Rmq() - snkbx = Snekbox() - rmq.consume(callback=snkbx.message_handler) - except KeyboardInterrupt: - print('Exited') - exit(0) [email protected]('/input', methods=["POST"]) +def code_input(): + body = request.get_json() + output = snekbox.python3(body["code"]) + return jsonify(input=body["code"], output=output) diff --git a/snekweb.py b/snekweb.py deleted file mode 100644 index ff1a72c..0000000 --- a/snekweb.py +++ /dev/null @@ -1,74 +0,0 @@ -import json -import logging -import threading -import traceback - -from flask import Flask, render_template -from flask_sockets import Sockets -from rmq import Rmq - -# Load app -app = Flask(__name__) -app.jinja_env.auto_reload = True -sockets = Sockets(app) - -# Logging -gunicorn_logger = logging.getLogger('gunicorn.error') -app.logger.handlers = gunicorn_logger.handlers -app.logger.setLevel(gunicorn_logger.level) -log = app.logger - - [email protected]('/') -def index(): - """Root path returns standard index.html.""" - - return render_template('index.html') - - [email protected]('/ws/<snekboxid>') -def websocket_route(ws, snekboxid): - """Opens a websocket that spawns and connects to a snekbox daemon.""" - - localdata = threading.local() - localdata.thread_ws = ws - - rmq = Rmq() - - def message_handler(ch, method, properties, body, thread_ws): - msg = body.decode('utf-8') - thread_ws.send(msg) - ch.basic_ack(delivery_tag=method.delivery_tag) - - consumer_parameters = {'queue': snekboxid, - 'callback': message_handler, - 'thread_ws': localdata.thread_ws} - - consumer = threading.Thread( - target=rmq.consume, - kwargs=consumer_parameters) - - consumer.daemon = True - consumer.start() - - try: - while not ws.closed: - message = ws.receive() - if message: - snek_msg = json.dumps({"snekid": snekboxid, "message": message}) - log.info(f"User {snekboxid} sends message\n{message.strip()}") - rmq.publish(snek_msg) - - except Exception: - log.info(traceback.format_exc()) - - finally: - if not ws.closed: - ws.close() - - -if __name__ == '__main__': - from gevent import pywsgi - from geventwebsocket.handler import WebSocketHandler - server = pywsgi.WSGIServer(('0.0.0.0', 5000), app, handler_class=WebSocketHandler) - server.serve_forever() diff --git a/templates/index.html b/templates/index.html index 8de9627..41980d1 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,106 +1,14 @@ <!DOCTYPE html> -<meta charset="utf-8" /> -<title>snekboxweb</title> -<script language="javascript" type="text/javascript"> - - -let _ready = false -let snekbox_id -var output; - -snekbox_id = sessionStorage.getItem("snekbox_id"); -console.log(snekbox_id) -if (snekbox_id == null) { - snekbox_id = generate_id() - sessionStorage.setItem("snekbox_id", snekbox_id) - console.log(snekbox_id) -} - -function init(){ - output = document.getElementById("output"); - websocketHandler(); -} - -function websocketHandler(){ - var here = window.location.host; - var wsUri = `ws://${here}/ws/`+snekbox_id; - websocket = new WebSocket(wsUri); - websocket.onopen = function(evt) { onOpen(evt) }; - websocket.onclose = function(evt) { onClose(evt) }; - websocket.onmessage = function(evt) { onMessage(evt) }; - websocket.onerror = function(evt) { onError(evt) }; -} - -function onOpen(evt){ - _ready = true - console.log("CONNECTED"); -} - -function onClose(evt){ - _ready = false - console.log("DISCONNECTED"); -} - -function onMessage(evt){ - writeToScreen('<span style="color: blue;">RESPONSE: ' + evt.data+'</span>'); -} - -function exit(){ - websocket.close(); -} - -function onError(evt){ - _ready = false - writeToScreen('<span style="color: red;">ERROR:</span> ' + evt.data); -} - -function sendMessage(msg){ - waitForSocketConnection(function(){ - websocket.send(msg); - }); - console.log("sent message "+msg) -} - -function waitForSocketConnection(callback){ - setTimeout( - function () { - if (_ready === true) { - if(callback != null){ - callback();} - return; - } - else { - waitForSocketConnection(callback);} - - }, 500); // milliseconds -} - -function writeToScreen(message){ - var pre = document.createElement("p"); - pre.style.wordWrap = "break-word"; - pre.innerHTML = message; - output.appendChild(pre); -} - -function sendFromInput(){ - var msg = document.getElementById("field1").value; - sendMessage(msg) -} - -function generate_id(){ - return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); -} - -window.addEventListener("load", init, false); - -</script> - -<textarea rows="4" cols="50" type="text" id="field1"> +<html> + <meta charset="utf-8" /> + <title>snekboxweb</title> + <body> + <form action="/result" method="POST"> + <p>Code:<br><textarea name="Code" rows="20" cols="80"> def sum(a,b): return a+b -print( sum(1,2) ) -</textarea> -<br> -<button onclick="sendFromInput()">Send</button> -<button onclick="exit()">disconnect from websocket</button> -<div id="output"></div> +print( sum(1,2) )</textarea></p> + <p><input type="submit" value="Run"></p> + </form> + </body> +</html> diff --git a/templates/result.html b/templates/result.html new file mode 100644 index 0000000..e339605 --- /dev/null +++ b/templates/result.html @@ -0,0 +1,9 @@ +<!DOCTYPE html> +<html> + <meta charset="utf-8" /> + <title>snekboxweb</title> + <body> + <p>Code Evaluated:<br><pre>{{ code }}</pre></p> + <p>Results:<br><pre>{{ result }}</pre></p> + </body> +</html> diff --git a/tests/test_snekbox.py b/tests/test_snekbox.py index e2505d6..cc79a2a 100644 --- a/tests/test_snekbox.py +++ b/tests/test_snekbox.py @@ -1,12 +1,6 @@ import unittest -import pytest -import os -import json from snekbox import Snekbox -from rmq import Rmq - -r = Rmq() snek = Snekbox() @@ -24,12 +18,14 @@ class SnekTests(unittest.TestCase): # self.assertEquals(result.strip(), 'timed out or memory limit exceeded') def test_timeout(self): - code = ('x = "*"\n' - 'while True:\n' - ' try:\n' - ' x = x * 99\n' - ' except:\n' - ' continue\n') + code = ( + 'x = "*"\n' + 'while True:\n' + ' try:\n' + ' x = x * 99\n' + ' except:\n' + ' continue\n' + ) result = snek.python3(code) self.assertEquals(result.strip(), 'timed out or memory limit exceeded') |