From 175fb38093f352269e365b1095a504baf9013f5a Mon Sep 17 00:00:00 2001 From: Nick Humrich Date: Fri, 28 Jul 2017 13:36:06 -0600 Subject: [PATCH] First real commit --- .dockerignore | 107 ++++++++++++++++++++++++ .gitignore | 105 ++++++++++++++++++++++++ Dockerfile | 12 +++ README.md | 100 +++++++++++++++++++++++ docker-compose.yml | 15 ++++ requirements.txt | 3 + run.py | 6 ++ smallprox/__init__.py | 0 smallprox/core.py | 20 +++++ smallprox/mapper.py | 90 ++++++++++++++++++++ smallprox/server.py | 186 ++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 644 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100755 run.py create mode 100644 smallprox/__init__.py create mode 100644 smallprox/core.py create mode 100644 smallprox/mapper.py create mode 100644 smallprox/server.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..29925e5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,107 @@ +.gitignore +.git/ +LICENSE +README.md + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24ba0a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,105 @@ +fullchain.pem +privkey.pem + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..35a960e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.6-alpine + +RUN apk add --no-cache -u gcc make musl-dev +RUN pip install dumb-init + +COPY requirements.txt /app/ +WORKDIR /app +RUN pip install -r requirements.txt + +COPY . /app + +CMD ["dumb-init", "python3", "-u", "/app/run.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..438044a --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# Small-Prox +A small local reverse proxy (such as nginx/haproxy) for handling many local docker containers. + +This proxy routes traffic to specific containers based on host or path. +It also allows you to route traffic to local ports, in case your not +using docker for some services (common for local dev). + +This proxy is intended to route traffic to specific services much like the +load balancer would on a real environment (production). It helps develop locally. + +**Note: This proxy/project is intended to ease local development. There is no +security/performance considerations made at all. I do not recommend using this +to route traffic anywhere except locally.** + +# How does it work? +The container listens on the docker socket and watches for containers to start up. +The containers have a label that specifies what host and path they want +to handle traffic for, and this proxy sends it to them on those conditions. + +# Getting started + +To run the container simply use: + +```bash +docker run -d --net host -v /var/run/docker.sock:/var/run/docker.sock nhumrich/small-prox +``` + +or use docker-compose: +```yaml +version: '3' +services: + smallprox: + image: nhumrich/small-prox + network_mode: host + volumes: + - /var/run/docker.sock:/var/run/docker.sock +``` + +### Forwarding to docker containers +Start your container with a label + +```bash +docker run -l localproxy_expose=myapp.myhost.local=8080 myorg/mycontainer +``` + +The format is: +`localproxy_expose={host}/{path}={port container listens on}` + +If you are doing host based networking, you will need to add the hosts in your +/etc/hosts file pointing to 127.0.0.1 or use an actual dns record that resolves +to 127.0.0.1 + +### Forwarding to local ports +If you would like to forward traffic to local ports, you can do this by setting +environment variables on the small-prox container before you start it. + +The environment variable is `LOCAL_PORTS` and excepts a comma-separated list of +strings in the format of `{host}/{path}={port}`. + + +#FAQ +### Do I need to use host networking (`--net host`)? + +Depends on how your using this. If you are only using this proxy to proxy to containers, + then you could just forward port 80 and 443. However, in order to use the + "local port forwarding" feature, you will need to run on the host network. + +### Can I use this in prod? +I mean, you *could*, but I dont recommend it. + +### Does it support ssl? +Sure does! You just need to drop a `fullchain.pem` and `privkey.pem` file into +the `/certs/` directory in the container and ssl will work. You could either volume mount these in +or build your own container on top of this one. The file names are the names letsencrypt uses. + You could use self-signed certs, or you could create a DNS name that points to 127.0.0.1 and use dns + validation to get a lets encrypt cert for your local dev. + +### Why did you build this? Why not just use jwilder/nginx-proxy (or similar)? +There are a couple reasons. `jwilder/nginx-proxy` is excellent but it only does + host routing. I wanted path based routing as well. Also, I wanted to be able to + route to local ports, for when i'm debugging locally and dont want to run my service inside a container. + +Another possibility is just use nginx by itself. This works great until you want to change things, such +as where a service is or what path/port it listens to. I was using a custom nginx image at my +organization, but ended up having many many versions of it for all the different configurations +people wanted. I found that I wanted the "configuration" outside of the container, and +in the persons repo. So, here is something a little more dynamic, and loads configuration from other places +(docker labels). Plus, now that its in python, it gives me more flexibility to add things in the future +if I want. + +### Does this use nginx/similar under the hood? +No. This project is written entirely in python. I had thought of just implementing it by +looking for docker changes, updating the nginx configuration and restarting nginx. +The amount of work (not much) to do that, is about the same to just listen to http and +forward packets. I decided to do it entirely in python as a learning experience for myself. +Since the project is intended for local development only, I am not concerned about +security/performance issues. +That being said, this project uses asyncio and one of pythons fastest http parsers, so you +shouldn't notice any slowness from it. + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0800efb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' +services: + localproxy: + build: . + environment: + LOCAL_PORTS: myexample.local=8080 + + network_mode: host + + volumes: + # add certs and uncomment to test ssl + #- ./fullchain.pem:/certs/fullchain.pem + #- ./privkey.pem:/certs/privkey.pem + - .:/app + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..09744b4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +httptools +aiodocker + diff --git a/run.py b/run.py new file mode 100755 index 0000000..01c021e --- /dev/null +++ b/run.py @@ -0,0 +1,6 @@ +from smallprox import core + + +core.main() + + diff --git a/smallprox/__init__.py b/smallprox/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smallprox/core.py b/smallprox/core.py new file mode 100644 index 0000000..4220f85 --- /dev/null +++ b/smallprox/core.py @@ -0,0 +1,20 @@ +import asyncio +import os + +from .server import HTTPServer +from .mapper import update_config, add_container + + + +def main(): + config = {} + loop = asyncio.get_event_loop() + local_ports = os.getenv('LOCAL_PORTS', []) + local_ports = local_ports and local_ports.split(',') + for port in local_ports: + add_container('local', port, config) + + server = HTTPServer(loop, config) + loop.run_until_complete(server.start()) + loop.create_task(update_config(config)) + loop.run_forever() diff --git a/smallprox/mapper.py b/smallprox/mapper.py new file mode 100644 index 0000000..e7dc9e2 --- /dev/null +++ b/smallprox/mapper.py @@ -0,0 +1,90 @@ +import asyncio + +import aiodocker + + +async def update_config(config: dict): + docker = aiodocker.Docker() + events = docker.events + subscriber = events.subscribe() + + asyncio.ensure_future(events.run()) + + for container in (await docker.containers.list()): + expose_label = container._container.get('Labels').get('localproxy_expose') + if expose_label: + add_container(container, expose_label, config) + + while True: + event = await subscriber.get() + status = event.get('status') + if status in ('start', 'die') and event.get('Type') == 'container': + print(event) + expose = event.get('Actor', {}).get('Attributes', {}).get('localproxy_expose') + if not expose: + # no expose label. Ignore container + continue + con = (await docker.containers.get( + event.get('Actor', {}).get('ID'))) + + if status == 'start': + add_container(con, expose, config) + elif status == 'die': + remove_container(con, expose, config) + + +def add_container(container, expose_label, config): + print(container) + print(expose_label) + + if container == 'local': + ip = '127.0.0.1' + else: + networks = container._container.get('NetworkSettings').get('Networks') + ip = networks.get('bridge').get('IPAddress') + + host, path, port = parse_expose_label(expose_label) + host_dict = config.get(host, {}) + host_dict[path] = f'{ip}:{port}' + config[host] = host_dict + + +def remove_container(container, expose_label, config): + host, path, port = parse_expose_label(expose_label) + host_dict = config.get(host) + if len(host_dict) == 1: + del config[host] + else: + del host_dict[path] + + +def parse_expose_label(expose_label): + url, port = expose_label.split('=') + if url.startswith('/'): + # url is only a path + host = '*' + path = 'url' + else: + # url contains a host + url_portions = url.split('/') + host = url_portions[0] + if len(url_portions) > 1: + path = url_portions[1] + else: + path = '' + return host, path, port + + +def get_host_and_port(host, path, config): + host_dict = config.get(host, {}) + all_hosts_dict = config.get('*', {}) + host_string = host_dict.get(path) + if not host_string: + host_string = all_hosts_dict.get(path) + if not host_string: + host_string = host_dict.get('') + if not host_string: + return None, None + else: + ip, port = host_string.split(':') + return ip, port diff --git a/smallprox/server.py b/smallprox/server.py new file mode 100644 index 0000000..e991b83 --- /dev/null +++ b/smallprox/server.py @@ -0,0 +1,186 @@ +import asyncio +import os +import traceback +from http import HTTPStatus +from collections import namedtuple +from ssl import SSLContext + +from .mapper import get_host_and_port + +from httptools import HttpRequestParser, HttpParserError, parse_url + + +Response = namedtuple('Response', 'status body headers') + + +class ClientConnection(asyncio.Protocol): + __slots__ = ('parent', 'loop', 'transport') + + def __init__(self, parent, loop): + self.parent = parent + self.loop = loop + self.transport = None + + def connection_made(self, transport): + self.transport = transport + + def send(self, data): + if not self.transport: + print('race') + else: + self.transport.write(data) + + def data_received(self, data): + self.parent.send_raw(data) + + def connection_lost(self, exc): + pass + + def close(self): + pass + + +class HTTPServer: + def __init__(self, loop, config): + self._loop = loop + self.config = config + self._server = None + self._redirect_server = None + + async def start(self): + ssl = None + port = 80 + if os.path.isfile('fullchain.pem'): + context = SSLContext() + context.load_cert_chain('fullchain.pem', 'privkey.pem') + ssl = context + + self._redirect_server = await self._loop.create_server( + lambda: _HTTPServerProtocol(loop=self._loop, + config=self.config, + ssl_forward=True), + host='0.0.0.0', + port=port + ) + port = 443 + + self._server = await self._loop.create_server( + lambda: _HTTPServerProtocol(loop=self._loop, config=self.config), + host='0.0.0.0', + port=port, + ssl=ssl + ) + print(f'Listening on port {port}') + + +class _HTTPServerProtocol(asyncio.Protocol): + """ HTTP Protocol handler. + Should only be used by HTTPServerTransport + """ + __slots__ = ('_transport', 'data', 'http_parser', + 'client', '_headers', '_url', 'config', 'ssl_forward') + + def __init__(self, *, loop, config, ssl_forward=False): + self.config = config + self._transport = None + self.data = None + self.http_parser = HttpRequestParser(self) + self.client = None + self._loop = loop + self._url = None + self._headers = None + self.ssl_forward = ssl_forward + + """ The next 3 methods are for asyncio.Protocol handling """ + def connection_made(self, transport): + self._transport = transport + self.client = None + self.data = b'' + + def connection_lost(self, exc): + if self.client: + self.client.close() + + def data_received(self, data): + try: + self.data += data + self.http_parser.feed_data(data) + self.send_data_to_client() + except HttpParserError as e: + traceback.print_exc() + self.send_response(Response(status=HTTPStatus.BAD_REQUEST, + body=b'invalid HTTP', + headers={})) + + """ + The following methods are for HTTP parsing (from httptools) + """ + def on_message_begin(self): + self._headers = {} + + def on_header(self, name, value): + key = name.decode('ASCII').lower() + val = value.decode() + self._headers[key] = val + + def on_headers_complete(self): + host = self._headers['host'] + host = host.split(':')[0] + url = parse_url(self._url) + + if self.ssl_forward: + self.send_response(Response(HTTPStatus.MOVED_PERMANENTLY, + body=b'Redirect to https', + headers={'Location': 'https://' + host + self._url.decode()})) + return + + ip, port = get_host_and_port(host, url.path, self.config) + if ip is None: + self.send_response(Response(HTTPStatus.SERVICE_UNAVAILABLE, + body=b'service unavailable', + headers={})) + return + self.client = ClientConnection(self, self._loop) + coro = self._loop.create_connection( + lambda: self.client, ip, int(port)) + task = self._loop.create_task(coro) + task.add_done_callback(self.send_data_to_client) + + def on_url(self, url): + self._url = url + + """ + End parsing methods + """ + + def send_data_to_client(self, future=None): + if self.client and self.client.transport: + self.client.send(self.data) + self.data = b'' + + def send_raw(self, data): + self._transport.write(data) + + def send_response(self, response): + headers = 'HTTP/1.1 {status_code} {status_message}\r\n'.format( + status_code=response.status.value, + status_message=response.status.phrase, + ) + headers += 'Connection: close\r\n' + + if response.body: + headers += 'Content-Type: text/plain\r\n' + headers += 'Content-Length: {}\r\n'.format(len(response.body)) + else: + headers += 'Content-Length: {}\r\n'.format(0) + + if response.headers: + for header, value in response.headers.items(): + headers += '{header}: {value}\r\n'.format(header=header, + value=value) + + result = headers.encode('ASCII') + b'\r\n' + if response.body: + result += response.body + + self.send_raw(result)