diff --git a/appscale/tools/appscale.py b/appscale/tools/appscale.py index 51e1bb80..e54c754c 100644 --- a/appscale/tools/appscale.py +++ b/appscale/tools/appscale.py @@ -113,7 +113,6 @@ class AppScale(): undeploy Removes from the current deployment. DATA ASSOCIATED WITH THE APPLICATION WILL BE LOST. - upgrade Upgrades AppScale code to its latest version. """ # Deprecated AppScaleFile arguments DEPRECATED_ASF_ARGS = ['n', 'scp', 'appengine', 'max_memory', 'min', 'max'] @@ -894,36 +893,3 @@ def register(self, deployment_id): AppScaleLogger.success( 'Registration complete for AppScale deployment {0}.' .format(deployment['name'])) - - def upgrade(self): - """ Allows users to upgrade to the latest version of AppScale.""" - contents_as_yaml = yaml.safe_load(self.read_appscalefile()) - - # Construct the appscale-upgrade command from argv and the contents of - # the AppScalefile. - command = [] - - if 'keyname' in contents_as_yaml: - command.append("--keyname") - command.append(contents_as_yaml['keyname']) - - if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True: - command.append("--verbose") - - if 'ips_layout' in contents_as_yaml: - command.append('--ips_layout') - command.append( - base64.b64encode(yaml.dump(contents_as_yaml['ips_layout']))) - - if 'login' in contents_as_yaml: - command.extend(['--login', contents_as_yaml['login']]) - - if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: - command.append('--test') - - options = ParseArgs(command, 'appscale-upgrade').args - options.ips = yaml.safe_load(base64.b64decode(options.ips_layout)) - options.terminate = False - options.clean = False - options.instance_type = contents_as_yaml.get('instance_type') - AppScaleTools.upgrade(options) diff --git a/appscale/tools/appscale_tools.py b/appscale/tools/appscale_tools.py index 0460b564..591884c8 100644 --- a/appscale/tools/appscale_tools.py +++ b/appscale/tools/appscale_tools.py @@ -2,8 +2,6 @@ from __future__ import absolute_import -import Queue -import datetime import getpass import json import os @@ -11,10 +9,7 @@ import shutil import socket import sys -import threading import time -import traceback -import urllib2 import uuid from collections import Counter from itertools import chain @@ -37,27 +32,10 @@ from appscale.tools.local_state import APPSCALE_VERSION, LocalState from appscale.tools.node_layout import NodeLayout from appscale.tools.remote_helper import RemoteHelper -from appscale.tools.version_helper import latest_tools_version from appscale.agents.factory import InfrastructureAgentFactory -def async_layout_upgrade(ip, keyname, script, error_bucket, verbose=False): - """ Run a command over SSH and place exceptions in a bucket. - - Args: - ip: A string containing and IP address. - keyname: A string containing the deployment keyname. - script: A string to run as a command over SSH. - error_bucket: A thread-safe queue. - verbose: A boolean indicating whether or not to log verbosely. - """ - try: - RemoteHelper.ssh(ip, keyname, script, verbose) - except ShellException as ssh_error: - error_bucket.put(ssh_error) - - MIN_FREE_DISK_DB = 40.0 MIN_FREE_DISK = 10.0 MIN_AVAILABLE_MEMORY = 7.0 @@ -109,26 +87,6 @@ class AppScaleTools(object): ADMIN_CAPABILITIES = "upload_app" - # AppScale repository location on an AppScale image. - APPSCALE_REPO = "~/appscale" - - - # Bootstrap command to run. - BOOTSTRAP_CMD = '{}/bootstrap.sh >> /var/log/appscale/bootstrap.log'.\ - format(APPSCALE_REPO) - - - # Command to run the upgrade script from /appscale/scripts directory. - UPGRADE_SCRIPT = "python " + APPSCALE_REPO + "/scripts/upgrade.py" - - - # Template used for GitHub API calls. - GITHUB_API = 'https://api.github.com/repos/{owner}/{repo}' - - - # Location of the upgrade status file on the remote machine. - UPGRADE_STATUS_FILE_LOC = '/var/log/appscale/upgrade-status-' - @classmethod def add_instances(cls, options): """Adds additional machines to an AppScale deployment. @@ -1255,210 +1213,3 @@ def update_queues(cls, source_location, keyname, project_id): secret_key = LocalState.get_secret_key(keyname) admin_client = AdminClient(load_balancer_ip, secret_key) admin_client.update_queues(version.project_id, queues) - - @classmethod - def upgrade(cls, options): - """ Upgrades the deployment to the latest AppScale version. - Args: - options: A Namespace that has fields for each parameter that can be - passed in via the command-line interface. - """ - node_layout = NodeLayout(options) - previous_ips = LocalState.get_local_nodes_info(options.keyname) - previous_node_list = node_layout.from_locations_json_list(previous_ips) - node_layout.nodes = previous_node_list - - latest_tools = APPSCALE_VERSION - try: - AppScaleLogger.log( - 'Checking if an update is available for appscale-tools') - latest_tools = latest_tools_version() - except (URLError, ValueError): - # Prompt the user if version metadata can't be fetched. - if not options.test: - response = raw_input( - 'Unable to check for the latest version of appscale-tools. Would ' - 'you like to continue upgrading anyway? (y/N) ') - if response.lower() not in ['y', 'yes']: - raise AppScaleException('Cancelled AppScale upgrade.') - - if latest_tools > APPSCALE_VERSION: - raise AppScaleException( - "There is a newer version ({}) of appscale-tools available. Please " - "upgrade the tools package before running 'appscale upgrade'.". - format(latest_tools)) - - master_ip = node_layout.head_node().public_ip - upgrade_version_available = cls.get_upgrade_version_available() - - current_version = RemoteHelper.get_host_appscale_version( - master_ip, options.keyname, options.verbose) - - # Don't run bootstrap if current version is later that the most recent - # public one. Covers cases of revoked versions/tags and ensures we won't - # try to downgrade the code. - if current_version >= upgrade_version_available: - AppScaleLogger.log( - 'AppScale is already up to date. Skipping code upgrade.') - AppScaleLogger.log( - 'Running upgrade script to check if any other upgrades are needed.') - cls.shut_down_appscale_if_running(options) - cls.run_upgrade_script(options, node_layout) - return - - cls.shut_down_appscale_if_running(options) - cls.upgrade_appscale(options, node_layout) - - @classmethod - def run_upgrade_script(cls, options, node_layout): - """ Runs the upgrade script which checks for any upgrades needed to be performed. - Args: - options: A Namespace that has fields for each parameter that can be - passed in via the command-line interface. - node_layout: A NodeLayout object for the deployment. - """ - timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') - - db_ips = [node.private_ip for node in node_layout.nodes - if node.is_role('db_master') or node.is_role('db_slave')] - zk_ips = [node.private_ip for node in node_layout.nodes - if node.is_role('zookeeper')] - - upgrade_script_command = '{script} --keyname {keyname} '\ - '--log-postfix {timestamp} '\ - '--db-master {db_master} '\ - '--zookeeper {zk_ips} '\ - '--database {db_ips} '\ - '--replication {replication}'.format( - script=cls.UPGRADE_SCRIPT, - keyname=options.keyname, - timestamp=timestamp, - db_master=node_layout.db_master().private_ip, - zk_ips=' '.join(zk_ips), - db_ips=' '.join(db_ips), - replication=node_layout.replication - ) - master_public_ip = node_layout.head_node().public_ip - - AppScaleLogger.log("Running upgrade script to check if any other upgrade is needed.") - # Run the upgrade command as a background process. - error_bucket = Queue.Queue() - threading.Thread( - target=async_layout_upgrade, - args=(master_public_ip, options.keyname, upgrade_script_command, - error_bucket, options.verbose) - ).start() - - last_message = None - while True: - # Check if the SSH thread has crashed. - try: - ssh_error = error_bucket.get(block=False) - AppScaleLogger.warn('Error executing upgrade script') - LocalState.generate_crash_log(ssh_error, traceback.format_exc()) - except Queue.Empty: - pass - - upgrade_status_file = cls.UPGRADE_STATUS_FILE_LOC + timestamp + ".json" - command = 'cat' + " " + upgrade_status_file - upgrade_status = RemoteHelper.ssh( - master_public_ip, options.keyname, command, options.verbose) - json_status = json.loads(upgrade_status) - - if 'status' not in json_status or 'message' not in json_status: - raise AppScaleException('Invalid status log format') - - if json_status['status'] == 'complete': - AppScaleLogger.success(json_status['message']) - break - - if json_status['status'] == 'inProgress': - if json_status['message'] != last_message: - AppScaleLogger.log(json_status['message']) - last_message = json_status['message'] - time.sleep(cls.SLEEP_TIME) - continue - - # Assume the message is an error. - AppScaleLogger.warn(json_status['message']) - raise AppScaleException(json_status['message']) - - @classmethod - def shut_down_appscale_if_running(cls, options): - """ Checks if AppScale is running and shuts it down as this is an offline upgrade. - Args: - options: A Namespace that has fields for each parameter that can be - passed in via the command-line interface. - """ - if os.path.exists(LocalState.get_secret_key_location(options.keyname)): - AppScaleLogger.warn("AppScale needs to be down for this upgrade. " - "Upgrade process could take a while and it is not reversible.") - - if not options.test: - response = raw_input( - 'Are you sure you want to proceed with shutting down AppScale to ' - 'continue the upgrade? (y/N) ') - if response.lower() not in ['y', 'yes']: - raise AppScaleException("Cancelled AppScale upgrade.") - - AppScaleLogger.log("Shutting down AppScale...") - cls.terminate_instances(options) - else: - AppScaleLogger.warn("Upgrade process could take a while and it is not reversible.") - - if options.test: - return - - response = raw_input( - 'Are you sure you want to proceed with the upgrade? (y/N) ') - if response.lower() not in ['y', 'yes']: - raise AppScaleException("Cancelled AppScale upgrade.") - - @classmethod - def upgrade_appscale(cls, options, node_layout): - """ Runs the bootstrap script on each of the remote machines. - Args: - options: A Namespace that has fields for each parameter that can be - passed in via the command-line interface. - node_layout: A NodeLayout object for the deployment. - """ - unique_ips = [node.public_ip for node in node_layout.nodes] - - AppScaleLogger.log("Upgrading AppScale code to the latest version on " - "these machines: {}".format(unique_ips)) - threads = [] - error_ips = [] - for ip in unique_ips: - t = threading.Thread(target=cls.run_bootstrap, args=(ip, options, error_ips)) - threads.append(t) - - for x in threads: - x.start() - - for x in threads: - x.join() - - if not error_ips: - cls.run_upgrade_script(options, node_layout) - - @classmethod - def run_bootstrap(cls, ip, options, error_ips): - try: - RemoteHelper.ssh(ip, options.keyname, cls.BOOTSTRAP_CMD, options.verbose) - AppScaleLogger.success( - 'Successfully updated and built AppScale on {}'.format(ip)) - except ShellException: - error_ips.append(ip) - AppScaleLogger.warn('Unable to upgrade AppScale code on {}.\n' - 'Please correct any errors listed in /var/log/appscale/bootstrap.log ' - 'on that machine and re-run appscale upgrade.'.format(ip)) - return error_ips - - @classmethod - def get_upgrade_version_available(cls): - """ Gets the latest release tag version available. - """ - github_api = cls.GITHUB_API.format(owner='AppScale', repo='appscale') - response = urllib2.urlopen('{}/tags'.format(github_api)) - tag_list = json.loads(response.read()) - return tag_list[0]['name'] diff --git a/appscale/tools/parse_args.py b/appscale/tools/parse_args.py index 099011d0..cdb84e78 100644 --- a/appscale/tools/parse_args.py +++ b/appscale/tools/parse_args.py @@ -475,16 +475,6 @@ def add_allowed_flags(self, function): help="the name of the AppController property to set") self.parser.add_argument('--property_value', help="the value of the AppController property to set") - elif function == "appscale-upgrade": - self.parser.add_argument('--keyname', '-k', default=self.DEFAULT_KEYNAME, - help="the keypair name to use") - self.parser.add_argument('--ips_layout', - help="a YAML file dictating the placement strategy") - self.parser.add_argument( - '--login_host', help='The public IP address of the head node') - self.parser.add_argument( - '--test', action='store_true', default=False, - help='Skips user input when upgrading deployment') else: raise SystemExit @@ -584,8 +574,6 @@ def validate_allowed_flags(self, function): pass elif function == "appscale-set-property": pass - elif function == "appscale-upgrade": - pass else: raise SystemExit diff --git a/appscale/tools/scripts/appscale.py b/appscale/tools/scripts/appscale.py index 3e9267a3..64f790ce 100644 --- a/appscale/tools/scripts/appscale.py +++ b/appscale/tools/scripts/appscale.py @@ -231,12 +231,6 @@ def main(): elif command in ["--version", "-v"]: print APPSCALE_VERSION sys.exit(0) - elif command == "upgrade": - try: - appscale.upgrade() - except Exception as exception: - LocalState.generate_crash_log(exception, traceback.format_exc()) - sys.exit(1) else: print(AppScale.USAGE) if command == "help": diff --git a/appscale/tools/scripts/upgrade.py b/appscale/tools/scripts/upgrade.py deleted file mode 100644 index 4dc8eb97..00000000 --- a/appscale/tools/scripts/upgrade.py +++ /dev/null @@ -1,26 +0,0 @@ -# General-purpose Python library imports -import sys -import traceback - - -# AppScale library imports -# Make sure we're on Python 2.6 or greater before importing any code -# that's incompatible with older versions. -from .. import version_helper -version_helper.ensure_valid_python_is_used() - - -from ..appscale_tools import AppScaleTools -from ..local_state import LocalState -from ..parse_args import ParseArgs - - -def main(): - """ Execute appscale-upgrade script. """ - options = ParseArgs(sys.argv[1:], "appscale-upgrade").args - try: - AppScaleTools.upgrade(options) - sys.exit(0) - except Exception as e: - LocalState.generate_crash_log(e, traceback.format_exc()) - sys.exit(1) diff --git a/appscale/tools/version_helper.py b/appscale/tools/version_helper.py index 73697f60..05f33da4 100644 --- a/appscale/tools/version_helper.py +++ b/appscale/tools/version_helper.py @@ -9,23 +9,7 @@ # First-party Python libraries -import json import sys -import urllib2 - -# The location of appscale-tools on PyPI. -PYPI_URL = 'https://pypi.python.org/pypi/appscale-tools' - - -def latest_tools_version(): - """ Fetches the latest tools version available on PyPI. - - Returns: - A string containing a version number. - """ - response = urllib2.urlopen('{}/json'.format(PYPI_URL)) - pypi_info = json.loads(response.read()) - return pypi_info['info']['version'] def ensure_valid_python_is_used(system=sys): diff --git a/setup.py b/setup.py index 795da769..7be345f6 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,6 @@ 'appscale-set-property=appscale.tools.scripts.set_property:main', 'appscale-terminate-instances=' + 'appscale.tools.scripts.terminate_instances:main', - 'appscale-upgrade=appscale.tools.scripts.upgrade:main', 'appscale-upload-app=appscale.tools.scripts.upload_app:main' ] },