diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..0833a98 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7.4 diff --git a/README.md b/README.md index 6018195..be870d0 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,26 @@ -#mysqlapi +# mysqlapi [![Build Status](https://secure.travis-ci.org/tsuru/mysqlapi.png?branch=master)](http://travis-ci.org/tsuru/mysqlapi) This is a service API for MySQL, used for [tsuru](https://github.com/tsuru/tsuru). -Installation +Installation on dedicated host ------------ In order to have mysql API ready to receive requests, we need some bootstrap stuff. -The first step is to install the dependencies. Let's use pip to do it: +**Requirements :** `Python 3.7` + +The first step is to install the dependencies. Let's use `pip to do it: $ pip install -r requirements.txt -Now we need to run syncdb: +Now we need to run a migration before serving: - $ python manage.py syncdb + $ python manage.py migrate -Exporting enviroment variable to set the settings location: +Exporting environment variable to set the settings location: $ export DJANGO_SETTINGS_MODULE=mysqlapi.settings @@ -30,7 +32,7 @@ There are three modes to configure the API usage behavior: - `shared`: this configuration forces all applications to share the same mysql installation, in this mode, mysql API will create a new user and a new - database when added/binded by an app. + database when added/binded by one app. - `dedicated (on-demmand)`: every app using mysql will have a single vm for it's usage, in this mode, mysql API will create a vm, install everything needed to run mysql based on a predefined AMI and create a user and password. @@ -47,11 +49,11 @@ Shared Configuration -------------------- To run the API in shared mode, is needed to have a mysql installed and export -two enviroment variables. +two environment variables. One variable is to set the mysql host. If the shared mysql database is installed in the sabe vm that the app is, you can use `localhost` for -``MYSQLAPI_SHARED_SERVER``, but you'll also need to set up a externally +``MYSQLAPI_SHARED_SERVER``, but you'll also need to set up an externally accessible endpoint to be used by the apps that are using the service: $ MYSQLAPI_SHARED_SERVER=mysqlhost.com @@ -61,6 +63,8 @@ accessible endpoint to be used by the apps that are using the service: Running the api --------------- +Mysql must be installed on the same machine, otherwise, set environment variables as suggested in the next section. + $ gunicorn wsgi -b 0.0.0.0:8888 @@ -69,7 +73,7 @@ Try your configuration You can try if the previous configuration worked using curl: - $> curl -d 'name=myapp' http://youmysqlapi.com/resources + $> curl -d 'name=myapp' http://youmysqlapi.com This call is the same as to ``tsuru service-add `` and will return 201 if everything goes ok. @@ -85,21 +89,31 @@ You can deploy `mysqlapi` as tsuru appplication. ### Install MySQL server (Debian/Ubuntu) -First you should have a MySQL server. In Debian/Ubuntu, use `apt-get` to install it. +First you should have a MySQL server. In Debian/Ubuntu, use `apt` to install it. + +**Mysql 8 (Newer version, default)** +```bash +$ sudo apt install mysql +``` +**Old Mysql 5.6 version** ```bash -$ sudo apt-get install mysql-server-5.6 +$ sudo apt install mysql-server-5.6 ``` During install the installation script will aks you the password for `root` user. +By default, `mysqlapi` uses Mysql 8. Set the environment variable `MSQL_5_VERSION_ENABLED=True` **to enable Mysql 5.6 version**. +```bash +$ tsuru env-set -a mysqlapi MSQL_5_VERSION_ENABLED=True +``` #### Create database for mysqlapi After install MySQL, you need to create a user and a database for `mysqlapi`, -that is needed to store informations about created instances. +that is needed to store information about created instances. -``` +```bash mysql -u root -p ``` @@ -107,8 +121,19 @@ Here is an example: ```sql CREATE DATABASE mysqlapi CHARACTER SET utf8 COLLATE utf8_bin; +``` + +*Mysql 8 (Newer version)* +```bash +CREATE USER 'mysqlapi'@'%' IDENTIFIED BY 'mysqlpass'; +GRANT ALL PRIVILEGES ON mysqlapi.* TO 'mysqlapi'@'%'; +``` + +*Old Mysql 5.6 version* +```sql GRANT ALL PRIVILEGES ON mysqlapi.* TO 'mysqlapi'@'%' IDENTIFIED BY 'mysqlpass'; ``` + Configure mysql to accept external connection, create a file `/etc/mysql/conf.d/bind.cnf` with the following content: ``` @@ -118,14 +143,14 @@ bind-address = 0.0.0.0 To finish restart MySQL server: -``` +```bash sudo service mysql restart ``` ### Install service -Now you can install `mysqlapi` service. In your tsuru client machine (with crane installed): +Now you can install `mysqlapi` service. In your tsuru client machine: ```bash $ git clone https://github.com/tsuru/mysqlapi @@ -141,15 +166,19 @@ $ tsuru env-set -a mysqlapi MYSQLAPI_DB_USER=mysqlapi $ tsuru env-set -a mysqlapi MYSQLAPI_DB_PASSWORD=mysqlpass $ tsuru env-set -a mysqlapi MYSQLAPI_DB_HOST=db.192.168.50.4.nip.io -# salt used to hash the username/password -$ tsuru env-set -a mysqlapi MYSQLAPI_SALT=****** - -# Exporting enviroment variable to set the settings location +# Exporting environment variable to set the settings location tsuru env-set -a mysqlapi DJANGO_SETTINGS_MODULE=mysqlapi.settings ``` Export these variables to specify the shared cluster: +In that configuration, **`root` user has to be accessible by the app**. If needed, create a new root user on Mysql with a secure ip range corresponding to your app machine ip. +Please, be careful of security issues concerned about those two commands : +```sql +CREATE USER 'root'@'%' IDENTIFIED BY 'rootpassword'; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; +``` + ```bash # these settings can be different with mysqlapi's database $ tsuru env-set -a mysqlapi MYSQLAPI_SHARED_SERVER=db.192.168.50.4.nip.io @@ -176,13 +205,23 @@ Configure the service template and point it to your application: $ tsuru app-info -a mysqlapi | grep Address # set production address $ editor service.yaml -$ crane create service.yaml +$ tsuru service-create service.yaml +``` + +Prepare for production: + +```bash +$ tsuru env-set -a mysqlapi MYSQLAPI_DEBUG=0 ``` To list your services: ```bash -$ crane list -# OR $ tsuru service-list ``` + +## Issue + +You can not bind an intance with a dash `-` into the name. Django 1.9 can't parse the name on the url `/resources/firstpart-secondpart/bind-app`. Resulting on a 404 error. + +Instead use underscore `_` in names. \ No newline at end of file diff --git a/mysqlapi/api/creator.py b/mysqlapi/api/creator.py index 0ccebf0..a20e7cd 100644 --- a/mysqlapi/api/creator.py +++ b/mysqlapi/api/creator.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import Queue +import queue import threading model_class = None @@ -11,7 +11,7 @@ class InstanceQueue(object): def __init__(self): - self._queue = Queue.Queue() + self._queue = queue.Queue() self._closed = False self._sem = threading.Semaphore() @@ -55,7 +55,7 @@ def run(self): while not _instance_queue.closed: try: instance = _instance_queue.get(timeout=2) - except Queue.Empty: + except queue.Empty: continue if not self.ec2_client.get(instance): _instance_queue.put(instance) diff --git a/mysqlapi/api/models.py b/mysqlapi/api/models.py index 904b507..bc71f1b 100644 --- a/mysqlapi/api/models.py +++ b/mysqlapi/api/models.py @@ -32,16 +32,17 @@ class DatabaseCreationError(Exception): pass -def generate_password(string): - return hashlib.sha1(string + settings.SALT).hexdigest() +def generate_password(): + return hashlib.sha1(str(os.urandom(256)).encode('utf-8')).hexdigest() -def generate_user(username): - if len(username) > 16: - _username = username[:12] + generate_password(username)[:4] +def generate_user(username, host): + userhost = username + '-' + host + if len(userhost) > 20: + _userhost = userhost[:20] else: - _username = username - return _username + _userhost = userhost + return (_userhost + '-' + hashlib.sha1(str(username + host).encode('utf-8')).hexdigest())[:32] class DatabaseManager(object): @@ -68,8 +69,12 @@ def public_host(self): def create_database(self): self.conn.open() cursor = self.conn.cursor() - sql = "CREATE DATABASE %s default character set utf8 " + \ - "default collate utf8_general_ci" + if settings.MSQL_5_VERSION_ENABLED: + sql = "CREATE DATABASE %s default character set utf8 " + \ + "default collate utf8_general_ci" + else: + sql = "CREATE DATABASE %s default character set utf8mb4 " + \ + "default collate utf8mb4_unicode_ci" cursor.execute(sql % self.name) self.conn.close() @@ -82,18 +87,25 @@ def drop_database(self): def create_user(self, username, host): self.conn.open() cursor = self.conn.cursor() - username = generate_user(username) - password = generate_password(username) - sql = ("grant all privileges on {0}.* to '{1}'@'%'" - " identified by '{2}'") - cursor.execute(sql.format(self.name, username, password)) + username = generate_user(username, host) + password = generate_password() + + if settings.MSQL_5_VERSION_ENABLED: + sql = ("grant all privileges on {0}.* to '{1}'@'%'" + " identified by '{2}'") + cursor.execute(sql.format(self.name, username, password)) + else: + sql = ("CREATE USER '{0}'@'%' IDENTIFIED BY '{1}';") + cursor.execute(sql.format(username, password)) + sql = ("GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'%';") + cursor.execute(sql.format(self.name, username)) self.conn.close() return username, password def drop_user(self, username, host): self.conn.open() cursor = self.conn.cursor() - username = generate_user(username) + username = generate_user(username, host) cursor.execute("drop user '{0}'@'%'".format(username)) self.conn.close() @@ -164,7 +176,7 @@ def db_manager(self): class ProvisionedInstance(models.Model): - instance = models.ForeignKey(Instance, null=True, blank=True, unique=True) + instance = models.OneToOneField(Instance, null=True, blank=True, on_delete = models.CASCADE) host = models.CharField(max_length=500) port = models.IntegerField(default=3306) admin_user = models.CharField(max_length=255, default="root") @@ -257,6 +269,6 @@ def _create_dedicate_database(instance, ec2_client): def canonicalize_db_name(name): if re.search(r"[\W\s]", name) is not None: - prefix = hashlib.sha1(name).hexdigest()[:10] + prefix = hashlib.sha1(str(name).encode('utf-8')).hexdigest()[:10] name = re.sub(r"[\W\s]", "_", name) + prefix return name diff --git a/mysqlapi/api/tests/test_models.py b/mysqlapi/api/tests/test_models.py index bf2376b..f81d0be 100644 --- a/mysqlapi/api/tests/test_models.py +++ b/mysqlapi/api/tests/test_models.py @@ -378,8 +378,6 @@ def test_canonicalize_db_name_do_nothing_when_called_twice(self): class GeneratePasswordTestCase(TestCase): - @override_settings(SALT="salt") def test_generate_password(self): - expected = hashlib.sha1("bla" + settings.SALT).hexdigest() - result = models.generate_password("bla") - self.assertEqual(expected, result) + result = models.generate_password() + self.assertIsNotNone(result) diff --git a/mysqlapi/api/views.py b/mysqlapi/api/views.py index 7a1f604..2b204e2 100644 --- a/mysqlapi/api/views.py +++ b/mysqlapi/api/views.py @@ -8,6 +8,7 @@ import subprocess from django.http import HttpResponse +from django.http import QueryDict from django.views.decorators.http import require_http_methods from django.views.generic.base import View @@ -31,9 +32,16 @@ def post(self, request, name, *args, **kwargs): msg = u"You can't bind to this instance because it's not running." return HttpResponse(msg, status=412) db = instance.db_manager() + + if "app-name" not in request.POST: + return HttpResponse("Instance app-name is missing", status=500) + appname = request.POST.get("app-name") + if not appname: + return HttpResponse("Instance app-name is empty", status=500) + try: - username, password = db.create_user(name, None) - except Exception, e: + username, password = db.create_user(name, appname) + except Exception as e: return HttpResponse(e.args[-1], status=500) config = { "MYSQL_HOST": db.public_host, @@ -51,9 +59,17 @@ def delete(self, request, name, *args, **kwargs): except Instance.DoesNotExist: return HttpResponse("Instance not found.", status=404) db = instance.db_manager() + + request_delete = QueryDict(request.body) + if "app-name" not in request_delete: + return HttpResponse("Instance app-name is missing", status=500) + appname = request_delete.get("app-name") + if not appname: + return HttpResponse("Instance app-name is empty", status=500) + try: - db.drop_user(name, None) - except Exception, e: + db.drop_user(name, appname) + except Exception as e: return HttpResponse(e.args[-1], status=500) return HttpResponse("", status=200) @@ -123,7 +139,7 @@ def export(request, name): try: db = DatabaseManager(name, host) return HttpResponse(db.export()) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: return HttpResponse(e.output.split(":")[-1].strip(), status=500) diff --git a/mysqlapi/settings.py b/mysqlapi/settings.py index df74b84..1bd1412 100644 --- a/mysqlapi/settings.py +++ b/mysqlapi/settings.py @@ -4,6 +4,8 @@ import os +import django + ROOT = os.path.abspath(os.path.dirname(__file__)) DEBUG = int(os.environ.get("MYSQLAPI_DEBUG", 1)) != 0 TEMPLATE_DEBUG = DEBUG @@ -165,8 +167,11 @@ S3_SECRET_KEY = os.environ.get("TSURU_S3_SECRET_KEY") S3_BUCKET = os.environ.get("TSURU_S3_BUCKET") -SALT = os.environ.get("MYSQLAPI_SALT", "") - ALLOWED_HOSTS = [ os.environ.get("MYSQLAPI_ALLOWED_HOST", "localhost"), ] + +MSQL_5_VERSION_ENABLED = os.environ.get("MSQL_5_VERSION_ENABLED", 'False') in \ + ("True", "true", "1") + +django.setup() diff --git a/requirements.apt b/requirements.apt index d4588d5..c211429 100644 --- a/requirements.apt +++ b/requirements.apt @@ -1,3 +1,3 @@ python-dev -libmysqlclient-dev +default-libmysqlclient-dev libevent-dev diff --git a/requirements.txt b/requirements.txt index 66fefb6..c0fed76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -Django==1.6 -MySQL-python==1.2.3 -boto==2.7.0 +Django==1.9 +mysqlclient +#MySQL-python==1.2.5 +boto crane-ec2==0.2.1 -gunicorn==0.14.6 -gevent==0.13.8 +gunicorn +gevent diff --git a/service.yaml b/service.yaml index 9de675e..7e8e82c 100644 --- a/service.yaml +++ b/service.yaml @@ -1,6 +1,6 @@ id: mysqlapi -password: mysql123 +username: mysqlapi +password: password team: admin endpoint: - production: mysqlapi.com - test: localhost:8000 + production: production-endpoint.com \ No newline at end of file diff --git a/tsuru.yaml b/tsuru.yaml index a196a7d..4e0170f 100644 --- a/tsuru.yaml +++ b/tsuru.yaml @@ -1,4 +1,6 @@ hooks: build: - - python manage.py syncdb --noinput + #- python manage.py syncdb --noinput + - python manage.py makemigrations --noinput + - python manage.py migrate --run-syncdb --noinput