From 35cb0aec23fc82a6aab50ab20159dd5964b0d23a Mon Sep 17 00:00:00 2001 From: Tanvi Marballi Date: Wed, 28 Feb 2018 11:30:46 -0800 Subject: [PATCH] Clean up after the load balancer nodes in case one of them has a problem while spawning. --- appscale/tools/remote_helper.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/appscale/tools/remote_helper.py b/appscale/tools/remote_helper.py index 0b5ca5ef..26044418 100644 --- a/appscale/tools/remote_helper.py +++ b/appscale/tools/remote_helper.py @@ -170,9 +170,23 @@ def start_all_nodes(cls, options, node_layout): instance_type_params = params.copy() instance_type_params['instance_type'] = instance_type - instance_ids, public_ips, private_ips = cls.spawn_nodes_in_cloud( - agent, instance_type_params, count=len(load_balancer_nodes), - load_balancer=True) + try: + instance_ids, public_ips, private_ips = cls.spawn_nodes_in_cloud( + agent, instance_type_params, count=len(load_balancer_nodes), + load_balancer=True) + except (AgentRuntimeException, BotoServerError): + AppScaleLogger.warn("AppScale was unable to start the requested number " + "of instances, attempting to terminate those that " + "were started.") + if len(spawned_instance_ids) > 0: + AppScaleLogger.warn("Attempting to terminate those that were started.") + cls.terminate_spawned_instances(spawned_instance_ids, agent, params) + + # Cleanup the keyname since it failed. + LocalState.cleanup_keyname(options.keyname) + + # Re-raise the original exception. + raise # Keep track of instances we have started. spawned_instance_ids.extend(instance_ids)