Merge pull request #857 from globocom/dev

Dev to Master

dbaas/backup/tasks.py

@@ -86,7 +86,7 @@ def unlock_instance(driver, instance, client):
         return False
 
 
-def make_instance_snapshot_backup(instance, error, group,
+def make_instance_old_snapshot_backup(instance, error, group,
                                   provider_class=VolumeProviderSnapshot,
                                   target_volume=None,
                                   current_hour=None,
@@ -211,6 +211,144 @@ def make_instance_snapshot_backup(instance, error, group,
 
     return snapshot
 
+def make_instance_snapshot_backup(
+    instance, error, group, provider_class=VolumeProviderSnapshot, target_volume=None,
+    current_hour=None, task=None, persist=0
+):
+    LOG.info("Make instance backup for {}".format(instance))
+    provider = provider_class(instance)
+    infra = instance.databaseinfra
+    database = infra.databases.first()
+
+    backup_retry_attempts = Configuration.get_by_name_as_int('backup_retry_attempts', default=3)
+
+    snapshot = Snapshot.create(
+        instance, group, target_volume or provider.volume,
+        environment=provider.environment, persistent=True if persist != 0 else False
+    )
+
+    snapshot_final_status = Snapshot.SUCCESS
+
+    locked = None
+    client = None
+    driver = infra.get_driver()
+    try:
+        client = driver.get_client(instance)
+        locked = lock_instance(driver, instance, client)
+        if not locked:
+            snapshot_final_status = Snapshot.WARNING
+
+        if 'MySQL' in type(driver).__name__:
+            mysql_binlog_save(client, instance)
+
+        has_snapshot = Snapshot.objects.filter(
+            status=Snapshot.WARNING, instance=instance, end_at__year=datetime.now().year,
+            end_at__month=datetime.now().month, end_at__day=datetime.now().day
+        )
+        backup_hour_list = Configuration.get_by_name_as_list('make_database_backup_hour')
+        if not snapshot_final_status == Snapshot.WARNING and not has_snapshot:
+            cont = 0
+            for _ in range(backup_retry_attempts):
+                cont += 1
+                try:
+                    code = 201
+                    response, data = provider.new_take_snapshot(persist=persist)
+
+                    if response.status_code < 400:
+                        break
+
+                    if cont >= 3:
+                        raise IndexError
+
+                except IndexError as e:
+                    content, response = e
+                    if response.status_code == 503:
+                        errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
+                            strftime("%d/%m/%Y %H:%M:%S"), instance
+                        )
+                        LOG.error(errormsg)
+                        if task:
+                            task.add_detail(errormsg)
+                        sleep(30)
+                    else:
+                        raise e
+
+            if response.status_code < 400:
+                while code != 200:
+                    sleep(20)
+                    snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
+                    if snap_response.status_code in [200, 202]:
+                        unlock_instance(driver, instance, client)
+                    if snap_response.status_code == 200:
+                        break
+                    if snap_response.status_code >= 400:
+                        raise error
+                    code = snap_response.status_code
+
+                snapshot.done(snap_status)
+                snapshot.save()
+            else:
+                errormsg = response['message']
+                set_backup_error(infra, snapshot, errormsg)
+        else:
+            if str(current_hour) in backup_hour_list:
+                raise Exception("Backup with WARNING already created today.")
+
+    except Exception as e:
+        errormsg = "Error creating snapshot: {}".format(e)
+        error['errormsg'] = errormsg
+        set_backup_error(infra, snapshot, errormsg)
+        return snapshot
+    finally:
+        unlock_instance(driver, instance, client)
+
+    if not snapshot.size:
+        command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
+            snapshot.snapshot_name
+        )
+        try:
+            output = instance.hostname.ssh.run_script(command)
+            size = int(output['stdout'][0])
+            snapshot.size = size
+        except Exception as e:
+            snapshot.size = 0
+            LOG.error("Error exec remote command {}".format(e))
+
+    backup_path = database.backup_path
+    if backup_path:
+        now = datetime.now()
+        target_path = "{}/{}/{}/{}/{}".format(
+            backup_path,
+            now.strftime("%Y_%m_%d"),
+            instance.hostname.hostname.split('.')[0],
+            now.strftime("%Y%m%d%H%M%S"),
+            infra.name
+        )
+        snapshot_path = "/data/.snapshot/{}/data/".format(
+            snapshot.snapshot_name
+        )
+        command = """
+        if [ -d "{backup_path}" ]
+        then
+            rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
+            mkdir -p {target_path}
+            cp -r {snapshot_path} {target_path} &
+        fi
+        """.format(backup_path=backup_path,
+                   target_path=target_path,
+                   snapshot_path=snapshot_path)
+        try:
+            instance.hostname.ssh.run_script(command)
+        except Exception as e:
+            LOG.error("Error exec remote command {}".format(e))
+
+    snapshot.status = snapshot_final_status
+    snapshot.end_at = datetime.now()
+    snapshot.save()
+    register_backup_dbmonitor(infra, snapshot)
+
+    return snapshot
+
 
 def make_instance_snapshot_backup_upgrade_disk(instance, error, group, provider_class=VolumeProviderSnapshot,
                                                target_volume=None,
@@ -742,6 +880,7 @@ def _create_database_backup(instance, task, group, current_hour, persist):
 
     error = {}
     try:
+        LOG.info('Starting make database snapshot')
         snapshot = make_instance_snapshot_backup(
             instance=instance,
             error=error,

dbaas/drivers/replication_topologies/base.py

@@ -732,6 +732,9 @@ def get_stop_database_vm_steps(self):
                 'workflow.steps.util.host_provider.StopIfRunning',
             )
         }]
+
+    def get_auto_upgrade_database_vm_offering(self):
+        raise NotImplementedError('Not implemented for topology')
 
     def get_start_database_vm_steps(self):
         return [{

dbaas/logical/views.py

@@ -2741,6 +2741,14 @@ def check_offering_sizes(request):
 @method_decorator(csrf_exempt)
 def resize_vm_from_btn(request, database_id, resize_target):    
     database = get_object_or_404(Database, pk=database_id)
+    user = request.user
+    from notification.tasks import TaskRegister
+
+    LOG.info("Starting database auto upgrade: database {}, user: {}".format(database, user))
+
+    TaskRegister.auto_upgrade_database_vm_offering(
+        database=database, user=user
+    )
 
     future_offering = database.get_future_offering(resize_target)
     return HttpResponse(json.dumps({'future_offering': future_offering.name}), content_type="application/json")

dbaas/maintenance/admin/__init__.py

@@ -28,6 +28,7 @@
 from . database_upgrade_disk_type import DatabaseUpgradeDiskTypeAdmin
 from . database_start_database_vm import DatabaseStartDatabaseVMAdmin
 from . database_stop_database_vm import DatabaseStopDatabaseVMAdmin
+from . database_auto_upgrade_vm_offering import DatabaseAutoUpgradeVMOferringAdmin
 
 
 admin.site.register(models.Maintenance, MaintenanceAdmin)
@@ -74,3 +75,7 @@
 admin.site.register(
     models.DatabaseStopDatabaseVM, DatabaseStopDatabaseVMAdmin
 )
+
+admin.site.register(
+    models.DatabaseAutoUpgradeVMOffering, DatabaseAutoUpgradeVMOferringAdmin
+)

dbaas/maintenance/admin/database_auto_upgrade_vm_offering.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+from django.conf.urls import patterns, url
+from django.contrib import messages
+from django.core.urlresolvers import reverse
+from django.http import HttpResponseRedirect
+from django.shortcuts import get_object_or_404
+from django.utils.html import format_html
+from .database_maintenance_task import DatabaseMaintenanceTaskAdmin
+from ..models import DatabaseAutoUpgradeVMOffering
+from notification.tasks import TaskRegister
+
+
+class DatabaseAutoUpgradeVMOferringAdmin(DatabaseMaintenanceTaskAdmin):
+    search_fields = (
+        "database__name", "database__databaseinfra__name", "task__id", "task__task_id"
+    )
+
+    list_display = (
+        "database", "database_team", "current_step", "current_step_class", "friendly_status",
+        "maintenance_action", "link_task", "started_at", "finished_at"
+    )
+
+    readonly_fields = (
+        "database", "task",
+        "started_at", "link_task", "finished_at", "status",
+        "maintenance_action", "task_schedule"
+    )
+
+    def maintenance_action(self, maintenance):
+        if not maintenance.is_status_error:
+            return 'N/A'
+
+        if not maintenance.can_do_retry:
+            return 'N/A'
+
+        url = "/admin/maintenance/databaseautoupgradevmoffering/{}/retry/".format(
+            maintenance.id
+        )
+        html = ("<a title='Retry' class='btn btn-info' "
+                "href='{}'>Retry</a>").format(url)
+        return format_html(html)
+
+    def get_urls(self):
+        base = super(DatabaseAutoUpgradeVMOferringAdmin, self).get_urls()
+
+        admin = patterns(
+            '',
+            url(
+                r'^/?(?P<auto_upgrade_vm_offering_id>\d+)/retry/$',
+                self.admin_site.admin_view(self.retry_view),
+                name="auto_upgrade_vm_offering_retry"
+            ),
+        )
+        return admin + base
+
+    def retry_view(self, request, auto_upgrade_vm_offering_id):
+        retry_from = get_object_or_404(DatabaseAutoUpgradeVMOffering, pk=auto_upgrade_vm_offering_id)
+
+        error = False
+        if not retry_from.is_status_error:
+            error = True
+            messages.add_message(
+                request, messages.ERROR,
+                "You can not do retry because auto upgrade database vm offering status is '{}'".format(
+                    retry_from.get_status_display()
+                ),
+            )
+
+        if not retry_from.can_do_retry:
+            error = True
+            messages.add_message(
+                request, messages.ERROR, "Auto Upgrade VM Offering retry is disabled"
+            )
+
+        if error:
+            return HttpResponseRedirect(
+                reverse(
+                    'admin:maintenance_databaseautoupgradevmoffering_change',
+                    args=(auto_upgrade_vm_offering_id,)
+                )
+            )
+
+        TaskRegister.auto_upgrade_database_vm_offering(
+            database=retry_from.database,
+            user=request.user,
+            retry_from=retry_from
+        )
+
+        url = reverse('admin:notification_taskhistory_changelist')
+        filter = "user={}".format(request.user.username)
+        return HttpResponseRedirect('{}?{}'.format(url, filter))