-
Notifications
You must be signed in to change notification settings - Fork 898
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Remove EvmDatabaseOps and related rake tasks
Well... mostly. A single (set of) method(s) is needed as the appliance_console uses this method to check the current database connection. This code could be moved into the console itself, but since it was released recently, this allows that code to still function until a new version not requiring it can be released.
- Loading branch information
1 parent
e26d0db
commit 0fa2846
Showing
3 changed files
with
48 additions
and
663 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,243 +1,7 @@ | ||
| $LOAD_PATH << File.expand_path(__dir__) | ||
| require 'util/postgres_admin' | ||
|
|
||
| require 'mount/miq_generic_mount_session' | ||
| require 'util/miq_object_storage' | ||
|
|
||
| class EvmDatabaseOps | ||
| include Vmdb::Logging | ||
| BACKUP_TMP_FILE = "/tmp/miq_backup".freeze | ||
| DUMP_TMP_FILE = "/tmp/miq_pg_dump".freeze | ||
|
|
||
| DEFAULT_OPTS = {:dbname => 'vmdb_production'} | ||
|
|
||
| def self.backup_destination_free_space(file_location) | ||
| require 'fileutils' | ||
| parent_directory = File.dirname(file_location) | ||
| FileUtils.mkdir_p(parent_directory) | ||
|
|
||
| free_space = begin | ||
| output = AwesomeSpawn.run!("df", :params => {:P => parent_directory}, :combined_output => true).output | ||
| data_line = output.split("\n")[1] if output.kind_of?(String) | ||
| data_line.split[3].to_i * 1024 if data_line | ||
| end | ||
|
|
||
| free_space || 0 | ||
| end | ||
|
|
||
| def self.database_size(opts) | ||
| PostgresAdmin.database_size(opts) | ||
| end | ||
|
|
||
| def self.validate_free_space(database_opts) | ||
| free_space = backup_destination_free_space(database_opts[:local_file]) | ||
| db_size = database_size(database_opts) | ||
| if free_space > db_size | ||
| _log.info("[#{database_opts[:dbname]}] with database size: [#{db_size} bytes], free space at [#{database_opts[:local_file]}]: [#{free_space} bytes]") | ||
| else | ||
| msg = "Destination location: [#{database_opts[:local_file]}], does not have enough free disk space: [#{free_space} bytes] for database of size: [#{db_size} bytes]" | ||
| _log.warn(msg) | ||
| defined?(::MiqEvent) && MiqEvent.raise_evm_event_queue(MiqServer.my_server, "evm_server_db_backup_low_space", :event_details => msg) | ||
| raise MiqException::MiqDatabaseBackupInsufficientSpace, msg | ||
| end | ||
| end | ||
|
|
||
| def self.backup(db_opts, connect_opts = {}) | ||
| # db_opts: | ||
| # :dbname => 'vmdb_production', | ||
| # :username => 'root', | ||
| # :local_file => "/tmp/backup_1", - Backup locally to the file specified | ||
|
|
||
| # connect_opts: | ||
| # :uri => "smb://dev005.manageiq.com/share1", | ||
| # :username => 'samba_one', | ||
| # :password => 'Zug-drep5s', | ||
| # :remote_file_name => "backup_1", - Provide a base file name for the uploaded file | ||
|
|
||
| uri = with_file_storage(:backup, db_opts, connect_opts) do |database_opts| | ||
| backup_result = PostgresAdmin.backup(database_opts) | ||
| backup_result | ||
| end | ||
| _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been backed up to file: [#{uri}]") | ||
| uri | ||
| end | ||
|
|
||
| def self.dump(db_opts, connect_opts = {}) | ||
| # db_opts and connect_opts similar to .backup | ||
|
|
||
| uri = with_file_storage(:dump, db_opts, connect_opts) do |database_opts| | ||
| PostgresAdmin.backup_pg_dump(database_opts) | ||
| end | ||
| _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been dumped up to file: [#{uri}]") | ||
| uri | ||
| end | ||
|
|
||
| def self.restore(db_opts, connect_opts = {}) | ||
| # db_opts: | ||
| # :local_file => "/tmp/backup_1", - Restore from this local file | ||
| # :dbname => 'vmdb_production' | ||
| # :username => 'root' | ||
|
|
||
| # connect_opts: | ||
| # :uri => "smb://dev005.manageiq.com/share1/db_backup/miq_pg_backup_20100719_215444", | ||
| # :username => 'samba_one', | ||
| # :password => 'Zug-drep5s', | ||
|
|
||
| uri = with_file_storage(:restore, db_opts, connect_opts) do |database_opts, backup_type| | ||
| prepare_for_restore(database_opts[:local_file], backup_type) | ||
|
|
||
| # remove all the connections before we restore; AR will reconnect on the next query | ||
| ActiveRecord::Base.connection_pool.disconnect! | ||
| PostgresAdmin.restore(database_opts.merge(:backup_type => backup_type)) | ||
| end | ||
| _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been restored from file: [#{uri}]") | ||
| uri | ||
| end | ||
|
|
||
| private_class_method def self.merged_db_opts(db_opts) | ||
| DEFAULT_OPTS.merge(db_opts) | ||
| end | ||
|
|
||
| STORAGE_ACTIONS_TO_METHODS = { :backup => :add, :dump => :add, :restore => :download }.freeze | ||
| private_class_method def self.with_file_storage(action, db_opts, connect_opts) | ||
| db_opts = merged_db_opts(db_opts) | ||
|
|
||
| if db_opts[:local_file].nil? | ||
| if action == :restore | ||
| uri = connect_opts[:uri] | ||
|
|
||
| connect_uri = URI::Generic.new(*URI.split(uri)) | ||
| connect_uri.path = File.dirname(connect_uri.path) | ||
| connect_opts[:uri] = connect_uri.to_s | ||
| else | ||
| connect_opts[:remote_file_name] ||= File.basename(backup_file_name(action)) | ||
| # | ||
| # If the passed in URI contains query parameters, ignore them | ||
| # when creating the dump file name. They'll be used in the session object. | ||
| # | ||
| uri_parts = [connect_opts[:uri].split('?')[0]] | ||
| uri_parts << (action == :dump ? "db_dump" : "db_backup") unless connect_opts[:skip_directory] | ||
| uri_parts << connect_opts[:remote_file_name] | ||
| uri = File.join(uri_parts) | ||
| end | ||
| else | ||
| uri = db_opts[:local_file] | ||
|
|
||
| # HACK(ish): This just puts the bare minimum necessary for URI.parse to | ||
| # recognize the :uri option as "file" scheme, and allows MiqFileStorage | ||
| # to then instantiate MiqLocalMountSession below in the | ||
| # `.with_interface_class` method. | ||
| connect_opts[:uri] = "file://" | ||
| end | ||
|
|
||
| MiqFileStorage.with_interface_class(connect_opts) do |file_storage| | ||
| send_args = [uri, db_opts[:byte_count]].compact | ||
|
|
||
| if action == :restore | ||
| # `MiqFileStorage#download` requires a `nil` passed to the block form | ||
| # to accommodate streaming | ||
| send_args.unshift(nil) | ||
| magic_numbers = { | ||
| :pgdump => PostgresAdmin::PG_DUMP_MAGIC, | ||
| :basebackup => PostgresAdmin::BASE_BACKUP_MAGIC | ||
| } | ||
| backup_type = file_storage.magic_number_for(uri, :accepted => magic_numbers) | ||
| end | ||
|
|
||
| # Note: `input_path` will always be a fifo stream (input coming from | ||
| # PostgresAdmin, and the output going to the `uri`), since we want to | ||
| # maintain the same interface for all backup types. | ||
| # | ||
| # This means that `uri` will always be the final destination, but | ||
| # `input_path` below will be an intermediary fifo that will take the | ||
| # input from `pg_dump`, `pg_restore`, or `pg_basebackup`, and streams the | ||
| # results from those commands (in ruby) it to whatever file storage | ||
| # endpoint `uri` is pointing to. | ||
| # | ||
| # This also makes sure that the streamed output is never written to disk | ||
| # locally, unless `uri` is targeting the local machine. This is why we | ||
| # set `db_opts` local file to that stream. | ||
| file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], *send_args) do |input_path| | ||
| db_opts[:local_file] = input_path | ||
| if action == :restore | ||
| yield(db_opts, backup_type) | ||
| else | ||
| if file_storage.class <= MiqGenericMountSession | ||
| # Only check free space on "mountable" storages | ||
| # | ||
| # For database dumps, this isn't going to be as accurate (since the | ||
| # dump size will probably be larger than the calculated DB size), but | ||
| # it still won't hurt to do as a generic way to get a rough idea if | ||
| # we have enough disk space on the appliance for the task. | ||
| free_space_opts = db_opts.merge(:local_file => file_storage.uri_to_local_path(uri)) | ||
| validate_free_space(free_space_opts) | ||
| end | ||
| yield(db_opts) | ||
| end | ||
| end | ||
| end | ||
|
|
||
| uri | ||
| end | ||
|
|
||
| private_class_method def self.prepare_for_restore(filename, backup_type = nil) | ||
| backup_type ||= validate_backup_file_type(filename) | ||
|
|
||
| if application_connections? | ||
| message = "Database restore failed. Shut down all evmserverd processes before attempting a database restore" | ||
| _log.error(message) | ||
| raise message | ||
| end | ||
|
|
||
| MiqRegion.replication_type = :none | ||
|
|
||
| connection_count = backup_type == :basebackup ? VmdbDatabaseConnection.unscoped.where(:backend_type => "client backend").count : VmdbDatabaseConnection.count | ||
| if connection_count > 1 | ||
| message = "Database restore failed. #{connection_count - 1} connections remain to the database." | ||
| _log.error(message) | ||
| raise message | ||
| end | ||
| end | ||
|
|
||
| private_class_method def self.validate_backup_file_type(filename) | ||
| if PostgresAdmin.base_backup_file?(filename) | ||
| :basebackup | ||
| elsif PostgresAdmin.pg_dump_file?(filename) | ||
| :pgdump | ||
| else | ||
| message = "#{filename} is not in a recognized database backup format" | ||
| _log.error(message) | ||
| raise message | ||
| end | ||
| end | ||
|
|
||
| private_class_method def self.application_connections? | ||
| VmdbDatabaseConnection.all.map(&:application_name).any? { |app_name| app_name.start_with?("MIQ") } | ||
| end | ||
|
|
||
| def self.gc(options = {}) | ||
| PostgresAdmin.gc(options) | ||
| end | ||
|
|
||
| def self.database_connections(database = nil, _type = :all) | ||
| database ||= ActiveRecord::Base.configurations[Rails.env]["database"] | ||
| conn = ActiveRecord::Base.connection | ||
| conn.client_connections.count { |c| c["database"] == database } | ||
| end | ||
|
|
||
| def self.upload(connect_opts, local_file, destination_file) | ||
| MiqGenericMountSession.in_depot_session(connect_opts) { |session| session.upload(local_file, destination_file) } | ||
| destination_file | ||
| end | ||
|
|
||
| def self.download(connect_opts, local_file) | ||
| MiqGenericMountSession.in_depot_session(connect_opts) { |session| session.download(local_file, connect_opts[:uri]) } | ||
| local_file | ||
| end | ||
|
|
||
| def self.backup_file_name(action = :backup) | ||
| time_suffix = Time.now.utc.strftime("%Y%m%d_%H%M%S") | ||
| "#{action == :backup ? BACKUP_TMP_FILE : DUMP_TMP_FILE}_#{time_suffix}" | ||
| end | ||
| private_class_method :backup_file_name | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.