From b4939383fdb37f75bcd33b000386328e130bd9a1 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Mon, 2 Aug 2021 18:41:28 -0500 Subject: [PATCH 1/2] Delete FileDepot/LogFile --- Gemfile | 1 - app/models/database_backup.rb | 99 ------ app/models/file_depot.rb | 37 --- app/models/file_depot/import_export.rb | 9 - app/models/file_depot_ftp.rb | 135 --------- app/models/file_depot_ftp_anonymous.rb | 9 - app/models/file_depot_nfs.rb | 13 - app/models/file_depot_s3.rb | 85 ------ app/models/file_depot_smb.rb | 21 -- app/models/file_depot_swift.rb | 71 ----- app/models/log_file.rb | 272 ----------------- app/models/miq_region.rb | 5 - app/models/miq_schedule.rb | 49 +-- app/models/miq_schedule/import_export.rb | 24 -- app/models/miq_server.rb | 1 - app/models/miq_server/log_management.rb | 278 ----------------- app/models/miq_task.rb | 1 - app/models/miq_task/purging.rb | 1 - app/models/mixins/file_depot_mixin.rb | 204 ------------- app/models/pxe_server.rb | 2 - app/models/zone.rb | 2 - config/replication_exclude_tables.yml | 1 - spec/factories/database_backup.rb | 5 - spec/factories/file_depot.rb | 15 - spec/factories/log_file.rb | 7 - spec/models/database_backup_spec.rb | 47 --- spec/models/file_depot_ftp_anonymous_spec.rb | 6 - spec/models/file_depot_ftp_spec.rb | 141 --------- spec/models/file_depot_nfs_spec.rb | 19 -- spec/models/file_depot_swift_spec.rb | 29 -- spec/models/log_collection_spec.rb | 127 -------- spec/models/log_file_spec.rb | 253 ---------------- spec/models/miq_schedule_filter_spec.rb | 12 - spec/models/miq_schedule_spec.rb | 229 -------------- spec/models/miq_server/log_management_spec.rb | 281 ------------------ spec/models/miq_task/purging_spec.rb | 4 - .../shared_examples_for_log_collection.rb | 58 ---- 37 files changed, 2 insertions(+), 2551 deletions(-) delete mode 100644 app/models/database_backup.rb delete mode 100644 app/models/file_depot.rb delete mode 100644 app/models/file_depot/import_export.rb delete mode 100644 app/models/file_depot_ftp.rb delete mode 100644 app/models/file_depot_ftp_anonymous.rb delete mode 100644 app/models/file_depot_nfs.rb delete mode 100644 app/models/file_depot_s3.rb delete mode 100644 app/models/file_depot_smb.rb delete mode 100644 app/models/file_depot_swift.rb delete mode 100644 app/models/log_file.rb delete mode 100644 app/models/miq_server/log_management.rb delete mode 100644 app/models/mixins/file_depot_mixin.rb delete mode 100644 spec/factories/database_backup.rb delete mode 100644 spec/factories/file_depot.rb delete mode 100644 spec/factories/log_file.rb delete mode 100644 spec/models/database_backup_spec.rb delete mode 100644 spec/models/file_depot_ftp_anonymous_spec.rb delete mode 100644 spec/models/file_depot_ftp_spec.rb delete mode 100644 spec/models/file_depot_nfs_spec.rb delete mode 100644 spec/models/file_depot_swift_spec.rb delete mode 100644 spec/models/log_collection_spec.rb delete mode 100644 spec/models/log_file_spec.rb delete mode 100644 spec/models/miq_server/log_management_spec.rb delete mode 100644 spec/support/examples_group/shared_examples_for_log_collection.rb diff --git a/Gemfile b/Gemfile index 6bd2129c72f..d40a801bd62 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,6 @@ gem "activerecord-session_store", "~>2.0" gem "activerecord-virtual_attributes", "~>3.0.0" gem "acts_as_tree", "~>2.7" # acts_as_tree needs to be required so that it loads before ancestry gem "ancestry", "~>4.1.0", :require => false -gem "aws-sdk-s3", "~>1.0", :require => false # For FileDepotS3 gem "bcrypt", "~> 3.1.10", :require => false gem "bundler", "~> 2.1", ">= 2.1.4", "!= 2.2.10", :require => false gem "byebug", :require => false diff --git a/app/models/database_backup.rb b/app/models/database_backup.rb deleted file mode 100644 index 9c392c29ed8..00000000000 --- a/app/models/database_backup.rb +++ /dev/null @@ -1,99 +0,0 @@ -class DatabaseBackup < ApplicationRecord - SUPPORTED_DEPOTS = %w[FileDepotSmb FileDepotNfs FileDepotS3 FileDepotSwift].freeze - - def self.supported_depots - SUPPORTED_DEPOTS.map { |model| [model, model.constantize.display_name] }.to_h - end - - def self.backup(options) - create.backup(options) - end - - def self.gc(options) - create.gc(options) - end - - def backup(options) - # TODO: Create a real exception out of this - unless options[:task_id].kind_of?(Integer) && options[:file_depot_id].kind_of?(Integer) - raise _("Missing or Invalid task: %{task_id}, depot id: %{depot_id}") % {:task_id => options[:task_id], - :depot_id => options[:file_depot_id]} - end - - task = MiqTask.find(options[:task_id]) - task.update_status("Active", "Ok", "Starting DB Backup for Region: #{region_name}") - - schedule_id = options[:miq_schedule_id] - @sch = MiqSchedule.find(schedule_id) if schedule_id.kind_of?(Integer) - - options[:userid] ||= "system" - - depot = FileDepot.find_by(:id => options[:file_depot_id]) - _backup(:uri => depot.uri, :username => depot.authentication_userid, :password => depot.authentication_password, :remote_file_name => backup_file_name, :region => depot.aws_region) - - if @sch && @sch.adhoc == true - _log.info("Removing adhoc schedule: [#{@sch.id}] [#{@sch.name}]") - @sch.destroy - end - - task.update_status("Finished", "Ok", "Completed DB Backup for Region: #{region_name}.") - task.id - end - - def _backup(options) - # add the metadata about this backup to this instance: (region, source hostname, db version, md5, status, etc.) - - connect_opts = options.slice(:uri, :username, :password, :region) - connect_opts[:remote_file_name] = options[:remote_file_name] if options[:remote_file_name] - EvmDatabaseOps.backup(current_db_opts, connect_opts) - end - - def gc(options) - unless options[:task_id].kind_of?(Integer) - raise _("Missing or Invalid task: %{task_id}") % {:task_id => options[:task_id]} - end - - task = MiqTask.find(options[:task_id]) - task.update_status("Active", "Ok", "Starting DB GC for Region: #{region_name}") - - options[:userid] ||= "system" - - EvmDatabaseOps.gc(current_db_opts.merge(options)) - task.update_status("Finished", "Ok", "Completed DB GC for Region: #{region_name}.") - task.id - end - - def restore(_options) - # Check PG - end - - def self.region_name - "region_#{my_region_number}" - end - - delegate :region_name, :to => :class - - def schedule_name - @schedule_name ||= begin - sch_name = @sch.name.gsub(/[^[:alnum:]]/, "_") if @sch - sch_name ||= "schedule_unknown" - sch_name - end - end - - def backup_file_name - File.join(region_name, schedule_name, "#{region_name}_#{Time.now.utc.strftime("%Y%m%d_%H%M%S")}.backup") - end - - private - - def current_db_opts - current = ActiveRecord::Base.configurations[Rails.env] - { - :hostname => current["host"], - :dbname => current["database"], - :username => current["username"], - :password => current["password"] - } - end -end diff --git a/app/models/file_depot.rb b/app/models/file_depot.rb deleted file mode 100644 index df495e4b87c..00000000000 --- a/app/models/file_depot.rb +++ /dev/null @@ -1,37 +0,0 @@ -class FileDepot < ApplicationRecord - include NewWithTypeStiMixin - include AuthenticationMixin - include_concern 'ImportExport' - include YAMLImportExportMixin - - has_many :miq_schedules, :dependent => :nullify - has_many :miq_servers, :dependent => :nullify, :foreign_key => :log_file_depot_id - has_many :log_files - validates_presence_of :uri - - attr_accessor :file - - def self.supported_depots - descendants.each_with_object({}) { |klass, hash| hash[klass.name] = klass.display_name } - end - - def self.supported_protocols - @supported_protocols ||= subclasses.each_with_object({}) { |klass, hash| hash[klass.uri_prefix] = klass.name }.freeze - end - - def self.requires_credentials? - true - end - - def requires_support_case? - false - end - - def upload_file(file) - @file = file - end - - def merged_uri(uri, _api_port) - uri - end -end diff --git a/app/models/file_depot/import_export.rb b/app/models/file_depot/import_export.rb deleted file mode 100644 index f18b149730d..00000000000 --- a/app/models/file_depot/import_export.rb +++ /dev/null @@ -1,9 +0,0 @@ -module FileDepot::ImportExport - extend ActiveSupport::Concern - - def export_to_array - export_attributes = attributes.except('id', 'created_at', 'updated_at') - export_attributes['AuthenticationsContent'] = authentications.map(&:export_to_array) - [self.class.to_s => export_attributes] - end -end diff --git a/app/models/file_depot_ftp.rb b/app/models/file_depot_ftp.rb deleted file mode 100644 index c2a00b3568f..00000000000 --- a/app/models/file_depot_ftp.rb +++ /dev/null @@ -1,135 +0,0 @@ -require 'net/ftp' - -class FileDepotFtp < FileDepot - attr_accessor :ftp - - def self.uri_prefix - "ftp" - end - - def self.validate_settings(settings) - new(:uri => settings[:uri]).verify_credentials(nil, settings.slice(:username, :password)) - end - - def upload_file(file) - super - with_connection do - begin - return if file_exists?(destination_file) - - upload(file.local_file, destination_file) - rescue => err - msg = "Error '#{err.message.chomp}', writing to FTP: [#{uri}], Username: [#{authentication_userid}]" - _log.error(msg) - raise _("Error '%{message}', writing to FTP: [%{uri}], Username: [%{id}]") % {:message => err.message.chomp, - :uri => uri, - :id => authentication_userid} - else - file.update( - :state => "available", - :log_uri => destination_file - ) - file.post_upload_tasks - end - end - end - - def remove_file(file) - @file = file - _log.info("Removing log file [#{destination_file}]...") - with_connection do |ftp| - ftp.delete(destination_file.to_s) - end - _log.info("Removing log file [#{destination_file}]...complete") - end - - def verify_credentials(_auth_type = nil, cred_hash = nil) - res = with_connection(cred_hash, &:last_response) - raise _("Depot Settings validation failed") unless res - res - end - - def with_connection(cred_hash = nil) - raise _("no block given") unless block_given? - _log.info("Connecting through #{self.class.name}: [#{name}]") - begin - connection = connect(cred_hash) - @ftp = connection - yield connection - ensure - connection.try(:close) - @ftp = nil - end - end - - def connect(cred_hash = nil) - host = URI(uri).hostname - - begin - _log.info("Connecting to #{self.class.name}: #{name} host: #{host}...") - @ftp = Net::FTP.new(host) - @ftp.passive = true # Use passive mode to avoid firewall issues see http://slacksite.com/other/ftp.html#passive - # @ftp.debug_mode = true if settings[:debug] # TODO: add debug option - creds = cred_hash ? [cred_hash[:username], cred_hash[:password]] : login_credentials - @ftp.login(*creds) - _log.info("Connected to #{self.class.name}: #{name} host: #{host}") - rescue SocketError => err - _log.error("Failed to connect. #{err.message}") - raise - rescue Net::FTPPermError => err - _log.error("Failed to login. #{err.message}") - raise - else - @ftp - end - end - - def file_exists?(file_or_directory) - !ftp.nlst(file_or_directory.to_s).empty? - rescue Net::FTPPermError - false - end - - def self.display_name(number = 1) - n_('FTP', 'FTPs', number) - end - - private - - def create_directory_structure(directory_path) - pwd = ftp.pwd - directory_path.to_s.split('/').each do |directory| - unless ftp.nlst.include?(directory) - _log.info("creating #{directory}") - ftp.mkdir(directory) - end - ftp.chdir(directory) - end - ftp.chdir(pwd) - end - - def upload(source, destination) - create_directory_structure(destination_path) - _log.info("Uploading file: #{destination} to File Depot: #{name}...") - ftp.putbinaryfile(source, destination.to_s) - _log.info("Uploading file: #{destination_file}... Complete") - end - - def destination_file - destination_path.join(file.destination_file_name).to_s - end - - def destination_path - base_path.join(file.destination_directory) - end - - def base_path - # uri: "ftp://ftp.example.com/incoming" => # - path = URI(URI::DEFAULT_PARSER.escape(uri)).path - Pathname.new(path) - end - - def login_credentials - [authentication_userid, authentication_password] - end -end diff --git a/app/models/file_depot_ftp_anonymous.rb b/app/models/file_depot_ftp_anonymous.rb deleted file mode 100644 index 80afcd19028..00000000000 --- a/app/models/file_depot_ftp_anonymous.rb +++ /dev/null @@ -1,9 +0,0 @@ -class FileDepotFtpAnonymous < FileDepotFtp - def login_credentials - ["anonymous", "anonymous"] - end - - def self.display_name(number = 1) - n_('Anonymous FTP', 'Anonymous FTPs', number) - end -end diff --git a/app/models/file_depot_nfs.rb b/app/models/file_depot_nfs.rb deleted file mode 100644 index d7b99f9ff33..00000000000 --- a/app/models/file_depot_nfs.rb +++ /dev/null @@ -1,13 +0,0 @@ -class FileDepotNfs < FileDepot - def self.requires_credentials? - false - end - - def self.uri_prefix - "nfs" - end - - def self.display_name(number = 1) - n_('NFS', 'NFS', number) - end -end diff --git a/app/models/file_depot_s3.rb b/app/models/file_depot_s3.rb deleted file mode 100644 index 379871514ed..00000000000 --- a/app/models/file_depot_s3.rb +++ /dev/null @@ -1,85 +0,0 @@ -class FileDepotS3 < FileDepot - attr_accessor :s3 - - def self.uri_prefix - "s3" - end - - def self.validate_settings(settings) - new(:uri => settings[:uri]).verify_credentials(nil, settings.slice(:username, :password)) - end - - def connect(options = {}) - require 'aws-sdk-s3' - - username = options[:username] || authentication_userid(options[:auth_type]) - password = options[:password] || authentication_password(options[:auth_type]) - # Note: The hard-coded aws_region will be removed after manageiq-ui-class implements region selection - aws_region = options[:region] || "us-east-1" - - $aws_log ||= Vmdb::Loggers.create_logger("aws.log") - - Aws::S3::Resource.new( - :access_key_id => username, - :secret_access_key => ManageIQ::Password.try_decrypt(password), - :region => aws_region, - :logger => $aws_log, - :log_level => :debug, - :log_formatter => Aws::Log::Formatter.new(Aws::Log::Formatter.default.pattern.chomp) - ) - end - - def with_depot_connection(options = {}) - raise _("no block given") unless block_given? - _log.info("Connecting through #{self.class.name}: [#{name}]") - yield connect(options) - end - - def verify_credentials(auth_type = nil, options = {}) - - connection_rescue_block do - # aws-sdk does Lazy Connections, so call a cheap function - with_depot_connection(options.merge(:auth_type => auth_type)) do |s3| - validate_connection(s3) - end - end - - true - end - - def validate_connection(connection) - connection_rescue_block do - connection.client.list_buckets - end - end - - def connection_rescue_block - yield - rescue => err - miq_exception = translate_exception(err) - raise unless miq_exception - - _log.log_backtrace(err) - _log.error("Error Class=#{err.class.name}, Message=#{err.message}") - raise miq_exception - end - - def translate_exception(err) - require 'aws-sdk-s3' - - case err - when Aws::S3::Errors::SignatureDoesNotMatch - MiqException::MiqHostError.new("SignatureMismatch - check your AWS Secret Access Key and signing method") - when Aws::S3::Errors::AuthFailure - MiqException::MiqHostError.new("Login failed due to a bad username or password.") - when Aws::Errors::MissingCredentialsError - MiqException::MiqHostError.new("Missing credentials") - else - MiqException::MiqHostError.new("Unexpected response returned from system: #{err.message}") - end - end - - def self.display_name(number = 1) - n_('AWS S3', 'AWS S3', number) - end -end diff --git a/app/models/file_depot_smb.rb b/app/models/file_depot_smb.rb deleted file mode 100644 index 541849ab0ae..00000000000 --- a/app/models/file_depot_smb.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'mount/miq_generic_mount_session' - -class FileDepotSmb < FileDepot - def self.uri_prefix - "smb" - end - - def self.validate_settings(settings) - res = MiqSmbSession.new(settings).verify - raise _("Depot Settings validation failed with error: %{error}") % {:error => res.last} unless res.first - res - end - - def verify_credentials(_auth_type = nil, cred_hash = nil) - self.class.validate_settings(cred_hash.merge(:uri => uri)) - end - - def self.display_name(number = 1) - n_('Samba', 'Sambas', number) - end -end diff --git a/app/models/file_depot_swift.rb b/app/models/file_depot_swift.rb deleted file mode 100644 index f773a952c8b..00000000000 --- a/app/models/file_depot_swift.rb +++ /dev/null @@ -1,71 +0,0 @@ -class FileDepotSwift < FileDepot - def self.uri_prefix - "swift" - end - - def self.validate_settings(settings) - new(:uri => settings[:uri]).verify_credentials(nil, settings.slice(:username, :password)) - end - - def connect(options = {}) - openstack_handle(options).connect(options) - end - - def openstack_handle(options = {}) - require 'manageiq/providers/openstack/legacy/openstack_handle' - @openstack_handle ||= begin - username = options[:username] || authentication_userid(options[:auth_type]) - password = options[:password] || authentication_password(options[:auth_type]) - uri = options[:uri] - address = URI(uri).host - port = URI(uri).port - - extra_options = { - :ssl_ca_file => ::Settings.ssl.ssl_ca_file, - :ssl_ca_path => ::Settings.ssl.ssl_ca_path, - :ssl_cert_store => OpenSSL::X509::Store.new - } - extra_options[:domain_id] = v3_domain_ident - extra_options[:service] = "Compute" - extra_options[:omit_default_port] = ::Settings.ems.ems_openstack.excon.omit_default_port - extra_options[:read_timeout] = ::Settings.ems.ems_openstack.excon.read_timeout - begin - OpenstackHandle::Handle.new(username, password, address, port, keystone_api_version, security_protocol, extra_options) - rescue => err - msg = "Error connecting to Swift host #{address}. #{err}" - logger.error(msg) - raise err, msg, err.backtrace - end - end - end - - def verify_credentials(auth_type = 'default', options = {}) - host = URI(options[:uri]).host - options[:auth_type] = auth_type - connect(options.merge(:auth_type => auth_type)) - rescue Excon::Errors::Unauthorized => err - msg = "Access to Swift host #{host} failed due to a bad username or password." - logger.error("#{msg} #{err}") - raise msg - rescue => err - msg = "Error connecting to Swift host #{host}. #{err}" - logger.error(msg) - raise err, msg, err.backtrace - end - - def merged_uri(uri, api_port) - uri = URI(uri) - uri.port = api_port.presence || 5000 - query_elements = [] - query_elements << "region=#{openstack_region}" if openstack_region.present? - query_elements << "api_version=#{keystone_api_version}" if keystone_api_version.present? - query_elements << "domain_id=#{v3_domain_ident}" if v3_domain_ident.present? - query_elements << "security_protocol=#{security_protocol}" if security_protocol.present? - uri.query = query_elements.join('&').presence - uri.to_s - end - - def self.display_name(number = 1) - n_('OpenStack Swift', 'OpenStack Swift', number) - end -end diff --git a/app/models/log_file.rb b/app/models/log_file.rb deleted file mode 100644 index 2571aee58f7..00000000000 --- a/app/models/log_file.rb +++ /dev/null @@ -1,272 +0,0 @@ -require 'net/ftp' -require 'uri' -require 'mount/miq_generic_mount_session' - -class LogFile < ApplicationRecord - belongs_to :resource, :polymorphic => true - belongs_to :file_depot - belongs_to :miq_task - - LOG_REQUEST_TIMEOUT = 30.minutes - - cattr_reader :log_request_timeout - - before_destroy :remove - - def relative_path_for_upload(loc_file) - server = resource - zone = server.zone - path = "#{zone.name}_#{zone.id}", "#{server.name}_#{server.id}" - date_string = "#{format_log_time(logging_started_on)}_#{format_log_time(logging_ended_on)}" - fname = "#{File.basename(loc_file, ".*").capitalize}_" - fname += "region_#{MiqRegion.my_region.region rescue "unknown"}_#{zone.name}_#{zone.id}_#{server.name}_#{server.id}_#{date_string}#{File.extname(loc_file)}" - dest = File.join("/", path, fname) - _log.info("Built relative path: [#{dest}] from source: [#{loc_file}]") - dest - end - - # Base is the URI defined by the user - # loc_file is the name of the original file - def build_log_uri(base_uri, loc_file) - scheme, userinfo, host, port, registry, path, opaque, query, fragment = URI.split(URI::DEFAULT_PARSER.escape(base_uri)) - - # Convert encoded spaces back to spaces - path.gsub!('%20', ' ') - - relpath = relative_path_for_upload(loc_file) - new_path = File.join("/", path, relpath) - uri = URI::HTTP.new(scheme, userinfo, host, port, registry, new_path, opaque, query, fragment).to_s - _log.info("New URI: [#{uri}] from base: [#{base_uri}], and relative path: [#{relpath}]") - uri - end - - def upload - # TODO: Currently the ftp code in the LogFile class has LogFile logic for the destination folders (evm_1/server_1) and builds these paths and copies the logs - # appropriately. To make all the various mechanisms work, we need to build a destination URI based on the input filename and pass this along - # so that the nfs, ftp, smb, etc. mechanism have very little LogFile logic and only need to know how decipher the URI and build the directories as appropraite. - raise _("LogFile local_file is nil") unless local_file - unless File.exist?(local_file) - raise _("LogFile local_file: [%{file_name}] does not exist!") % {:file_name => local_file} - end - raise _("Log Depot settings not configured") unless file_depot - - method = get_post_method(file_depot.uri) - send("upload_log_file_#{method}") - end - - def remove - method = get_post_method(log_uri) - return if method.nil? - return send("remove_log_file_#{method}") if respond_to?("remove_log_file_#{method}") - - # At this point ftp should have returned - klass = Object.const_get("Miq#{method.capitalize}Session") - klass.new(legacy_depot_hash).remove(log_uri) - rescue Exception => err - _log.warn("#{err.message}, deleting #{log_uri} from FTP") - end - - def file_exists? - return true if log_uri.nil? - - method = get_post_method(log_uri) - return true if method.nil? - return send("file_exists_#{method}?") if respond_to?("file_exists_#{method}?") - - # At this point ftp should have returned - klass = Object.const_get("Miq#{method.capitalize}Session") - klass.new(legacy_depot_hash).exist?(log_uri) - end - - # main UI method to call to request logs from a server - def self.logs_from_server(*args) - options = args.extract_options! - userid = args[0] || "system" - - # If no server provided, use the MiqServer receiving this request - server = args[1] || MiqServer.my_server - - # All server types who provide logs must implement the following instance methods: - # - my_zone: which returns the zone in which they reside - # - who_am_i: which returns a log friendly string of the server's class and id - [:my_zone, :who_am_i].each { |meth| raise "#{meth} not implemented for #{server.class.name}" unless server.respond_to?(meth) } - zone = server.my_zone - resource = server.who_am_i - - _log.info("Queueing the request by userid: [#{userid}] for logs from server: [#{resource}]") - - begin - # Create the task for the UI to check - task = MiqTask.create(:name => "Zipped log retrieval for [#{resource}]", :userid => userid, :miq_server_id => server.id) - - # callback only on exceptions.. ie, on errors... second level callback will set status to finished - cb = {:class_name => task.class.name, :instance_id => task.id, :method_name => :queue_callback_on_exceptions, :args => ['Finished']} - - # Queue the async fetch of the logs from the server - specifying a timeout, the zone to process this request, and a callback - options = options.merge(:taskid => task.id, :klass => server.class.name, :id => server.id) - - MiqQueue.put( - :class_name => name, - :method_name => "_request_logs", - :args => [options], - :zone => zone, - :miq_callback => cb, - :msg_timeout => LOG_REQUEST_TIMEOUT, - :priority => MiqQueue::HIGH_PRIORITY - ) - rescue => err - task.queue_callback_on_exceptions('Finished', 'error', err.to_s, nil) if task - raise - else - # return task id to the UI - msg = "Queued the request for logs from server: [#{resource}]" - task.update_status("Queued", "Ok", msg) - _log.info("Task: [#{task.id}] #{msg}") - task.id - end - end - - def self.historical_logfile - empty_logfile(true) - end - - def self.current_logfile - empty_logfile(false) - end - - def self.empty_logfile(historical) - LogFile.create(:state => "collecting", - :historical => historical, - :description => "Default logfile") - end - - def self.ping_timeout - ::Settings.log.collection.ping_depot_timeout - end - - def self.do_ping? - ::Settings.log.collection.ping_depot == true - end - - def upload_log_file_ftp - file_depot.upload_file(self) - end - - def upload_log_file_nfs - uri_to_add = build_log_uri(file_depot.uri, local_file) - uri = MiqNfsSession.new(legacy_depot_hash).add(local_file, uri_to_add) - update( - :state => "available", - :log_uri => uri - ) - post_upload_tasks - end - - def upload_log_file_smb - uri_to_add = build_log_uri(file_depot.uri, local_file) - uri = MiqSmbSession.new(legacy_depot_hash).add(local_file, uri_to_add) - update( - :state => "available", - :log_uri => uri - ) - post_upload_tasks - end - - def remove_log_file_ftp - file_depot.remove_file(self) - end - - def destination_directory - File.join("#{resource.zone.name}_#{resource.zone.id}", "#{resource.name}_#{resource.id}") - end - - def self.logfile_name(resource, category = "Current", date_string = nil) - region = MiqRegion.my_region.try(:region) || "unknown" - [category, "region", region, resource.zone.name, resource.zone.id, resource.name, resource.id, date_string].compact.join(" ") - end - - def destination_file_name - name.gsub(/\s+/, "_").concat(File.extname(local_file)) - end - - def name - super || self.class.logfile_name(resource) - end - - def post_upload_tasks - FileUtils.rm_f(local_file) if File.exist?(local_file) - end - - def format_log_time(time) - time.respond_to?(:strftime) ? time.strftime("%Y%m%d_%H%M%S") : "unknown" - end - - private - - def get_post_method(uri) - return nil if uri.nil? - - # Convert all backslashes in the URI to forward slashes - uri.tr!('\\', '/') - - # Strip any leading and trailing whitespace - uri.strip! - - URI.split(URI::DEFAULT_PARSER.escape(uri))[0] - end - - def legacy_depot_hash - # TODO: Delete this and make FileDepotSmb and FileDepotNfs implement all of their upload/delete/etc. logic - { - :uri => file_depot.uri, - :username => file_depot.authentication_userid, - :password => file_depot.authentication_password, - } - end - - def self._request_logs(options) - taskid = options[:taskid] - klass = options.delete(:klass).to_s - id = options.delete(:id) - - log_header = "Task: [#{taskid}]" - - server = Object.const_get(klass).find(id) - resource = server.who_am_i - - # server must implement an instance method: started_on? which returns whether the server is started - unless server.respond_to?(:started?) - raise MiqException::Error, _("started? not implemented for %{server_name}") % {:server_name => server.class.name} - end - unless server.started? - if server.respond_to?(:name) - raise MiqException::Error, - _("Log request failed since [%{resource} %{server_name}] is not started") % {:resource => resource, - :server_name => server.name} - else - raise MiqException::Error, - _("Log request failed since [%{resource}] is not started") % {:resource => resource} - end - end - - task = MiqTask.find(taskid) - - msg = "Requesting logs from server: [#{resource}]" - _log.info("#{log_header} #{msg}") - task.update_status("Active", "Ok", msg) - - cb = {:class_name => task.class.name, :instance_id => task.id, :method_name => :queue_callback_on_exceptions, :args => ['Finished']} - unless server.respond_to?(:_post_my_logs) - raise MiqException::Error, - _("_post_my_logs not implemented for %{server_name}") % {:server_name => server.class.name} - end - options = options.merge(:callback => cb, :timeout => LOG_REQUEST_TIMEOUT) - server._post_my_logs(options) - - msg = "Requested logs from: [#{resource}]" - _log.info("#{log_header} #{msg}") - task.update_status("Queued", "Ok", msg) - end - - private_class_method :_request_logs -end diff --git a/app/models/miq_region.rb b/app/models/miq_region.rb index 34dabcb937f..83bf9bb3b06 100644 --- a/app/models/miq_region.rb +++ b/app/models/miq_region.rb @@ -5,7 +5,6 @@ class MiqRegion < ApplicationRecord has_many :metric_rollups, :as => :resource # Destroy will be handled by purger has_many :vim_performance_states, :as => :resource # Destroy will be handled by purger - virtual_has_many :database_backups, :class_name => "DatabaseBackup" virtual_has_many :ext_management_systems, :class_name => "ExtManagementSystem" virtual_has_many :hosts, :class_name => "Host" virtual_has_many :storages, :class_name => "Storage" @@ -34,10 +33,6 @@ class MiqRegion < ApplicationRecord PERF_ROLLUP_CHILDREN = [:ext_management_systems, :storages] - def database_backups - DatabaseBackup.in_region(region_number) - end - def ext_management_systems ExtManagementSystem.in_region(region_number) end diff --git a/app/models/miq_schedule.rb b/app/models/miq_schedule.rb index e1efb87b592..6bf72e60fa4 100644 --- a/app/models/miq_schedule.rb +++ b/app/models/miq_schedule.rb @@ -8,7 +8,7 @@ class MiqSchedule < ApplicationRecord validates :name, :uniqueness_when_changed => {:scope => [:userid, :resource_type]} validates :name, :description, :resource_type, :run_at, :presence => true - validate :validate_run_at, :validate_file_depot + validate :validate_run_at before_save :set_start_time_and_prod_default @@ -16,7 +16,6 @@ class MiqSchedule < ApplicationRecord virtual_column :v_zone_name, :type => :string, :uses => :zone virtual_column :next_run_on, :type => :datetime - belongs_to :file_depot belongs_to :miq_search belongs_to :resource, :polymorphic => true belongs_to :zone @@ -41,7 +40,7 @@ class MiqSchedule < ApplicationRecord SYSTEM_SCHEDULE_CLASSES = %w(MiqReport MiqAlert MiqWidget).freeze VALID_INTERVAL_UNITS = %w(minutely hourly daily weekly monthly once).freeze - ALLOWED_CLASS_METHOD_ACTIONS = %w(db_backup db_gc automation_request).freeze + ALLOWED_CLASS_METHOD_ACTIONS = %w(db_gc automation_request).freeze IMPORT_CLASS_NAMES = %w[MiqSchedule].freeze default_value_for :userid, "system" @@ -247,19 +246,6 @@ def action_automation_request(_klass, _at) AutomationRequest.create_from_scheduled_task(user, filter[:uri_parts], parameters) end - def action_db_backup(klass, _at) - self.sched_action ||= {} - self.sched_action[:options] ||= {} - self.sched_action[:options][:userid] = userid - opts = self.sched_action[:options] - opts[:file_depot_id] = file_depot.id - opts[:miq_schedule_id] = id - queue_opts = {:class_name => klass.name, :method_name => "backup", :args => [opts], :role => "database_operations", - :msg_timeout => ::Settings.task.active_task_timeout.to_i_with_method} - task_opts = {:action => "Database backup", :userid => self.sched_action[:options][:userid]} - MiqTask.generic_action_with_callback(task_opts, queue_opts) - end - def action_db_gc(klass, _at) self.sched_action ||= {} self.sched_action[:options] ||= {} @@ -274,10 +260,6 @@ def run_automation_request action_automation_request(AutomationRequest, nil) end - def run_adhoc_db_backup - action_db_backup(DatabaseBackup, nil) - end - def action_evaluate_alert(obj, _at) MiqAlert.evaluate_queue(obj) _log.info("Action [#{name}] has been run for target type: [#{obj.class}] with name: [#{obj.name}]") @@ -325,33 +307,6 @@ def validate_run_at end end - def validate_file_depot # TODO: Do we need this if the validations are on the FileDepot classes? - if self.sched_action.kind_of?(Hash) && self.sched_action[:method] == "db_backup" && file_depot - errors.add(:file_depot, "is missing credentials") if !file_depot.uri.to_s.starts_with?("nfs") && file_depot.missing_credentials? - errors.add(:file_depot, "is missing uri") if file_depot.uri.blank? - end - end - - def verify_file_depot(params) # TODO: This logic belongs in the UI, not sure where - depot_class = FileDepot.supported_protocols[params[:uri_prefix]] - depot = file_depot.class.name == depot_class ? file_depot : build_file_depot(:type => depot_class) - depot.name = params[:name] - uri = params[:uri] - api_port = params[:swift_api_port] - depot.aws_region = params[:aws_region] - depot.openstack_region = params[:openstack_region] - depot.keystone_api_version = params[:keystone_api_version] - depot.v3_domain_ident = params[:v3_domain_ident] - depot.security_protocol = params[:security_protocol] - depot.uri = api_port.blank? ? uri : depot.merged_uri(URI(uri), api_port) - if params[:save] - file_depot.save! - file_depot.update_authentication(:default => {:userid => params[:username], :password => params[:password]}) if (params[:username] || params[:password]) && depot.class.requires_credentials? - elsif depot.class.requires_credentials? - depot.verify_credentials(nil, params) - end - end - def next_interval_time unless self.valid? || errors[:run_at].blank? _log.warn("Invalid schedule [#{id}] [#{name}]: #{Array.wrap(errors[:run_at]).join(", ")}") diff --git a/app/models/miq_schedule/import_export.rb b/app/models/miq_schedule/import_export.rb index 9d4188cd859..55a6bf91eac 100644 --- a/app/models/miq_schedule/import_export.rb +++ b/app/models/miq_schedule/import_export.rb @@ -19,8 +19,6 @@ def handle_attributes(export_attributes) export_attributes['MiqSearchContent'] = MiqSearch.find_by(:id => export_attributes['miq_search_id']).export_to_array if export_attributes['miq_search_id'] - export_attributes['FileDepotContent'] = FileDepot.find_by(:id => export_attributes['file_depot_id']).export_to_array if export_attributes['file_depot_id'] - if export_attributes['resource_id'] schedule_resource = export_attributes["resource_type"].safe_constantize.find_by(:id => export_attributes['resource_id']) export_attributes['resource_name'] = schedule_resource&.name @@ -71,7 +69,6 @@ def import_from_hash(miq_schedule, options = nil) filter_resource_name = miq_schedule.delete("filter_resource_name") miq_search = miq_schedule.delete("MiqSearchContent") - file_depot = miq_schedule.delete("FileDepotContent") resource_name = miq_schedule.delete("resource_name") was_new_record = new_or_existing_schedule.new_record? @@ -95,27 +92,6 @@ def import_from_hash(miq_schedule, options = nil) schedule_attributes[:miq_search_id] = search.id end - if file_depot - authentication_content = file_depot[0].values[0].delete("AuthenticationsContent") - fd = FileDepot.where(file_depot[0].values[0]).first_or_create - - authentication_content[0].each do |auth| - x = auth.values[0] - x['resource'] = fd - - group_description = x.delete('miq_group_description') - miq_group = MiqGroup.find_by(:description => group_description) || User.find_by(:userid=>'admin').current_group - x['miq_group_id'] = miq_group.id - - tenant_name = x.delete('tenant_name') - tenant = Tenant.find_by(:name => tenant_name) || Tenant.tenant_root - x['tenant_id'] = tenant.id - - Authentication.where(x).first_or_create - end - new_or_existing_schedule.file_depot_id = fd.id - end - if resource_name schedule_resource = miq_schedule["resource_type"].safe_constantize.find_by(:name => resource_name) schedule_attributes['resource_id'] = schedule_resource.id if schedule_resource diff --git a/app/models/miq_server.rb b/app/models/miq_server.rb index 11a172daa5d..98f0bcfc2e8 100644 --- a/app/models/miq_server.rb +++ b/app/models/miq_server.rb @@ -7,7 +7,6 @@ class MiqServer < ApplicationRecord include_concern 'ServerSmartProxy' include_concern 'ConfigurationManagement' include_concern 'EnvironmentManagement' - include_concern 'LogManagement' include_concern 'QueueManagement' include_concern 'RoleManagement' include_concern 'StatusManagement' diff --git a/app/models/miq_server/log_management.rb b/app/models/miq_server/log_management.rb deleted file mode 100644 index 54df0eefdaf..00000000000 --- a/app/models/miq_server/log_management.rb +++ /dev/null @@ -1,278 +0,0 @@ -require 'util/postgres_admin' - -module MiqServer::LogManagement - extend ActiveSupport::Concern - - included do - belongs_to :log_file_depot, :class_name => "FileDepot" - has_many :log_files, :dependent => :destroy, :as => :resource - end - - def _post_my_logs(options) - # Make the request to the MiqServer whose logs are needed - MiqQueue.create_with( - :miq_callback => options.delete(:callback), - :msg_timeout => options.delete(:timeout), - :priority => MiqQueue::HIGH_PRIORITY, - :args => [options] - ).put_unless_exists( - :class_name => self.class.name, - :instance_id => id, - :method_name => "post_logs", - :server_guid => guid, - :zone => my_zone, - ) do |msg| - _log.info("Previous adhoc log collection is still running, skipping...Resource: [#{self.class.name}], id: [#{id}]") unless msg.nil? - nil - end - end - - def synchronize_logs(*args) - options = args.extract_options! - args << self unless args.last.kind_of?(self.class) - LogFile.logs_from_server(*args, options) - end - - def last_log_sync_on - log_files.maximum(:updated_on) - end - - def last_log_sync_message - last_log = log_files.order(:updated_on => :desc).first - last_log.try(:miq_task).try!(:message) - end - - def include_automate_models_and_dialogs?(value) - return value unless value.nil? - Settings.log.collection.include_automate_models_and_dialogs - end - - def post_logs(options) - taskid = options[:taskid] - task = MiqTask.find(taskid) - context_log_depot = log_depot(options[:context]) - - # the current queue item and task must be errored out on exceptions so re-raise any caught errors - raise _("Log depot settings not configured") unless context_log_depot - context_log_depot.update(:support_case => options[:support_case].presence) - - if include_automate_models_and_dialogs?(options[:include_automate_models_and_dialogs]) - post_automate_models(taskid, context_log_depot) - post_automate_dialogs(taskid, context_log_depot) - end - - post_historical_logs(taskid, context_log_depot) unless options[:only_current] - post_current_logs(taskid, context_log_depot) - task.update_status("Finished", "Ok", "Log files were successfully collected") - end - - def current_log_patterns - # use an array union to add pg log path patterns if not already there - ::Settings.log.collection.current.pattern | pg_log_patterns - end - - def pg_data_dir - PostgresAdmin.data_directory - end - - def pg_log_patterns - pg_data = pg_data_dir - return [] unless pg_data - - pg_data = Pathname.new(pg_data) - [pg_data.join("*.conf"), pg_data.join("pg_log/*"), Pathname.new("/etc/manageiq/postgresql.conf.d/*")] - end - - def log_start_and_end_for_pattern(pattern) - evm = VMDB::Util.get_evm_log_for_date(pattern) - return if evm.nil? - - VMDB::Util.get_log_start_end_times(evm) - end - - def format_log_time(time) - time.respond_to?(:strftime) ? time.strftime("%Y%m%d_%H%M%S") : "unknown" - end - - def log_patterns(log_type, base_pattern = nil) - case log_type.to_s.downcase - when "archive" - Array(::Settings.log.collection.archive.pattern).unshift(base_pattern) - when "current" - current_log_patterns - else - [base_pattern] - end - end - - def post_one_log_pattern(pattern, logfile, log_type) - task = logfile.miq_task - log_prefix = "Task: [#{task.id}]" - - log_start, log_end = if logfile.logging_started_on - [logfile.logging_started_on, logfile.logging_ended_on] - else - log_start_and_end_for_pattern(pattern) - end - - date_string = "#{format_log_time(log_start)} #{format_log_time(log_end)}" unless log_start.nil? && log_end.nil? - - msg = "Zipping and posting #{log_type.downcase} logs for [#{who_am_i}] from: [#{log_start}] to [#{log_end}]" - _log.info("#{log_prefix} #{msg}") - task.update_status("Active", "Ok", msg) - - begin - local_file = VMDB::Util.zip_logs(log_type.to_s.downcase.concat(".zip"), log_patterns(log_type, pattern), "system") - self.log_files << logfile - - logfile.update( - :local_file => local_file, - :logging_started_on => log_start, - :logging_ended_on => log_end, - :name => LogFile.logfile_name(self, log_type, date_string), - :description => "Logs for Zone #{zone.name rescue nil} Server #{self.name} #{date_string}", - ) - - logfile.upload - rescue StandardError, Timeout::Error => err - _log.error("#{log_prefix} Posting of #{log_type.downcase} logs failed for #{who_am_i} due to error: [#{err.class.name}] [#{err}]") - task.update_status("Finished", "Error", "Posting of #{log_type.downcase} logs failed for #{who_am_i} due to error: [#{err.class.name}] [#{err}]") - logfile.update(:state => "error") - raise - ensure - FileUtils.rm_f(local_file) if local_file && File.exist?(local_file) - end - msg = "#{log_type} log files from #{who_am_i} are posted" - _log.info("#{log_prefix} #{msg}") - task.update_status("Active", "Ok", msg) - end - - def post_automate_models(taskid, log_depot) - domain_zip = Rails.root.join("log", "domain.zip") - backup_automate_models(domain_zip) - now = Time.zone.now - - logfile = LogFile.historical_logfile - logfile.update(:file_depot => log_depot, - :miq_task => MiqTask.find(taskid), - :logging_started_on => now, - :logging_ended_on => now) - post_one_log_pattern(domain_zip, logfile, "Models") - ensure - FileUtils.rm_rf(domain_zip) - end - - def backup_automate_models(domain_zip) - Dir.chdir(Rails.root) do - MiqAeDatastore.backup('zip_file' => domain_zip, 'overwrite' => false) - end - end - - def post_automate_dialogs(taskid, log_depot) - dialog_directory = Rails.root.join("log", "service_dialogs") - FileUtils.mkdir_p(dialog_directory) - backup_automate_dialogs(dialog_directory) - now = Time.zone.now - - logfile = LogFile.historical_logfile - logfile.update(:file_depot => log_depot, - :miq_task => MiqTask.find(taskid), - :logging_started_on => now, - :logging_ended_on => now) - post_one_log_pattern(dialog_directory.join("*"), logfile, "Dialogs") - ensure - FileUtils.rm_rf(dialog_directory) - end - - def backup_automate_dialogs(dialog_directory) - Dir.chdir(Rails.root) do - TaskHelpers::Exports::ServiceDialogs.new.export(:keep_spaces => false, :directory => dialog_directory) - end - end - - def post_historical_logs(taskid, log_depot) - task = MiqTask.find(taskid) - log_prefix = "Task: [#{task.id}]" - log_type = "Archive" - - # Post all compressed logs for a specific date + configs, creating a new row per day - VMDB::Util.compressed_log_patterns.each do |pattern| - log_start, log_end = log_start_and_end_for_pattern(pattern) - date_string = "#{format_log_time(log_start)}_#{format_log_time(log_end)}" unless log_start.nil? && log_end.nil? - - cond = {:historical => true, :name => LogFile.logfile_name(self, log_type, date_string), :state => 'available'} - cond[:logging_started_on] = log_start unless log_start.nil? - cond[:logging_ended_on] = log_end unless log_end.nil? - logfile = log_files.find_by(cond) - - if logfile && logfile.log_uri.nil? - _log.info("#{log_prefix} #{log_type} logfile already exists with id: [#{logfile.id}] for [#{who_am_i}] with contents from: [#{log_start}] to: [#{log_end}]") - next - else - logfile = LogFile.historical_logfile - end - - logfile.update(:file_depot => log_depot, :miq_task => task) - post_one_log_pattern(pattern, logfile, log_type) - end - end - - def post_current_logs(taskid, log_depot) - delete_old_requested_logs - - logfile = LogFile.current_logfile - logfile.update(:file_depot => log_depot, :miq_task => MiqTask.find(taskid)) - post_one_log_pattern("log/*.log", logfile, "Current") - end - - def delete_old_requested_logs - log_files.where(:historical => false).destroy_all - end - - def delete_active_log_collections_queue - MiqQueue.create_with(:priority => MiqQueue::HIGH_PRIORITY).put_unless_exists( - :class_name => self.class.name, - :instance_id => id, - :method_name => "delete_active_log_collections", - :server_guid => guid - ) do |msg| - _log.info("Previous cleanup is still running, skipping...") unless msg.nil? - end - end - - def delete_active_log_collections - log_files.each do |lf| - if lf.state == 'collecting' - _log.info("Deleting #{lf.description}") - lf.miq_task&.(:state => 'Finished', :status => 'Error', :message => 'Log Collection Incomplete during Server Startup') - lf.destroy - end - end - - # Since a task is created before a logfile, there's a chance we have a task without a logfile - MiqTask.where(:miq_server_id => id).where("name like ?", "Zipped log retrieval for %").where("state != ?", "Finished").each do |task| - task.update(:state => 'Finished', :status => 'Error', :message => 'Log Collection Incomplete during Server Startup') - end - end - - def log_collection_active_recently?(since = nil) - since ||= 15.minutes.ago.utc - return true if log_files.exists?(["created_on > ? AND state = ?", since, "collecting"]) - MiqTask.exists?(["miq_server_id = ? and name like ? and state != ? and created_on > ?", id, "Zipped log retrieval for %", "Finished", since]) - end - - def log_collection_active? - return true if log_files.exists?(:state => "collecting") - MiqTask.exists?(["miq_server_id = ? and name like ? and state != ?", id, "Zipped log retrieval for %", "Finished"]) - end - - def log_depot(context) - context == "Zone" ? zone.log_file_depot : log_file_depot - end - - def base_zip_log_name - t = Time.now.utc.strftime('%FT%H_%M_%SZ'.freeze) - # Name the file based on GUID and time. GUID and Date/time of the request are as close to unique filename as we're going to get - "App-#{guid}-#{t}" - end -end diff --git a/app/models/miq_task.rb b/app/models/miq_task.rb index 6a389b5589c..09b057a6c93 100644 --- a/app/models/miq_task.rb +++ b/app/models/miq_task.rb @@ -30,7 +30,6 @@ class MiqTask < ApplicationRecord MESSAGE_TASK_COMPLETED_SUCCESSFULLY = 'Task completed successfully'.freeze MESSAGE_TASK_COMPLETED_UNSUCCESSFULLY = 'Task did not complete successfully'.freeze - has_one :log_file, :dependent => :destroy has_one :binary_blob, :as => :resource, :dependent => :destroy has_one :miq_report_result has_one :job, :dependent => :destroy diff --git a/app/models/miq_task/purging.rb b/app/models/miq_task/purging.rb index 82ad8d33ffa..f0339a92ddb 100644 --- a/app/models/miq_task/purging.rb +++ b/app/models/miq_task/purging.rb @@ -18,7 +18,6 @@ def purge_scope(older_than) def purge_associated_records(ids) Job.where(:miq_task_id => ids).delete_all - LogFile.where(:miq_task_id => ids).delete_all BinaryBlob.where(:resource_type => 'MiqTask', :resource_id => ids).delete_all end end diff --git a/app/models/mixins/file_depot_mixin.rb b/app/models/mixins/file_depot_mixin.rb deleted file mode 100644 index d14888cdced..00000000000 --- a/app/models/mixins/file_depot_mixin.rb +++ /dev/null @@ -1,204 +0,0 @@ -require 'uri' -require 'mount/miq_generic_mount_session' - -module FileDepotMixin - extend ActiveSupport::Concern - SUPPORTED_DEPOTS = { - 'smb' => 'Samba', - 'nfs' => 'Network File System', - 's3' => 'Amazon Web Services', - 'swift' => 'OpenStack Swift' - }.freeze - - included do - include AuthenticationMixin - before_save :verify_uri_prefix_before_save - end - - module ClassMethods - def verify_depot_settings(settings) - return true unless MiqEnvironment::Command.is_appliance? - - settings["password"] ||= find(settings["id"]).authentication_password if settings["id"] - res = mnt_instance(settings).verify - raise _("Connection Settings validation failed with error: %{error}") % {:error => res.last} unless res.first - res - end - - def verify_depot_settings_queue(userid, zone, options) - task_opts = { - :action => "Verify #{display_name} Credentials", - :userid => userid - } - - queue_opts = { - :class_name => name, - :method_name => "verify_depot_settings", - :args => [options], - :zone => zone - } - - MiqTask.generic_action_with_callback(task_opts, queue_opts) - end - - def mnt_instance(settings) - settings[:uri_prefix] ||= get_uri_prefix(settings[:uri]) - klass = "Miq#{settings[:uri_prefix].capitalize}Session".constantize - klass.new(settings) - end - - def get_uri_prefix(uri_str) - return nil if uri_str.nil? - - # Convert all backslashes in the URI to forward slashes - uri_str.tr!('\\', '/') - - # Strip any leading and trailing whitespace - uri_str.strip! - - scheme, _userinfo, _host, _port, _registry, _path, _opaque, _query, _fragment = URI.split(URI::DEFAULT_PARSER.escape(uri_str)) - scheme - end - end - - def requires_credentials? - case uri_prefix - when 'nfs' - false - else - true - end - end - - def validate_depot_credentials - # This only checks that credentials are present - errors.add(:file_depot, "is missing credentials") if self.requires_credentials? && self.missing_credentials? - end - - def verify_depot_credentials(_auth_type = nil) - self.class.verify_depot_settings(depot_settings(true)) - end - - def depot_settings(reload = false) - return @depot_settings if !reload && @depot_settings - @depot_settings = { - :uri => uri, - :uri_prefix => uri_prefix, - :username => authentication_userid, - :password => authentication_password - } - end - - def mnt - raise _("No credentials defined") if requires_credentials? && missing_credentials? - - return @mnt if @mnt - @mnt = self.class.mnt_instance(depot_settings) - end - - # - # API methods - # - - def connect_depot - @connected ||= 0 - mnt.connect if @connected == 0 - @connected += 1 - end - - def disconnect_depot - @connected ||= 0 - return if @connected == 0 - mnt.disconnect if @connected == 1 - @connected -= 1 - end - alias_method :close, :disconnect_depot # TODO: Do we still need this alias? Since this is a mixin, close is a bad override. - - def with_depot - connect_depot - yield - ensure - disconnect_depot - end - - def depot_root - with_depot do - mnt.mnt_point - end - end - - def file_exists?(file) - with_depot do - !mnt.glob(file).empty? - end - end - - def file_glob(pattern) - with_depot do - mnt.glob(pattern) - end - end - - def file_stat(file) - with_depot do - mnt.stat(file) - end - end - - def file_read(file) - with_depot do - mnt.read(file) - end - end - - def file_write(file, contents) - with_depot do - mnt.write(file, contents) - end - end - - def file_delete(file) - with_depot do - mnt.delete(file) - end - end - alias_method :directory_delete, :file_delete - - def file_open(*args, &block) - with_depot do - mnt.open(*args, &block) - end - end - - def file_add(source, dest_uri) - with_depot do - mnt.add(source, dest_uri) - end - end - - def file_remove(uri) - with_depot do - mnt.remove(uri) - end - end - - def file_download(local_file, remote_file) - with_depot do - mnt.download(local_file, remote_file) - end - end - - def file_file?(file) - with_depot do - mnt.file?(file) - end - end - - # - # Callback methods - # - - def verify_uri_prefix_before_save - self.uri_prefix = self.class.get_uri_prefix(uri) - end -end diff --git a/app/models/pxe_server.rb b/app/models/pxe_server.rb index a78bd46abfa..2eef5c0ddda 100644 --- a/app/models/pxe_server.rb +++ b/app/models/pxe_server.rb @@ -1,8 +1,6 @@ class PxeServer < ApplicationRecord autoload :WimParser, "wim_parser" - include FileDepotMixin - alias_attribute :description, :name default_value_for :customization_directory, "" diff --git a/app/models/zone.rb b/app/models/zone.rb index c8913a215fd..d08337a649f 100644 --- a/app/models/zone.rb +++ b/app/models/zone.rb @@ -4,8 +4,6 @@ class Zone < ApplicationRecord serialize :settings, Hash - belongs_to :log_file_depot, :class_name => "FileDepot" - has_many :miq_servers has_many :ext_management_systems has_many :paused_ext_management_systems, :class_name => 'ExtManagementSystem', :foreign_key => :zone_before_pause_id diff --git a/config/replication_exclude_tables.yml b/config/replication_exclude_tables.yml index 9efdcfc4e2c..5aa47f56bd6 100644 --- a/config/replication_exclude_tables.yml +++ b/config/replication_exclude_tables.yml @@ -13,7 +13,6 @@ - conditions_miq_policies - custom_buttons - customization_specs -- database_backups - event_logs - external_urls - file_depots diff --git a/spec/factories/database_backup.rb b/spec/factories/database_backup.rb deleted file mode 100644 index 51a1c7fe2ef..00000000000 --- a/spec/factories/database_backup.rb +++ /dev/null @@ -1,5 +0,0 @@ -FactoryBot.define do - factory :database_backup do - sequence(:name) { |val| "db_backup_#{val}" } - end -end diff --git a/spec/factories/file_depot.rb b/spec/factories/file_depot.rb deleted file mode 100644 index 685ee697861..00000000000 --- a/spec/factories/file_depot.rb +++ /dev/null @@ -1,15 +0,0 @@ -FactoryBot.define do - factory :file_depot do - name { "File Depot" } - uri { "nfs://somehost/export" } - end - - factory(:file_depot_ftp, :class => "FileDepotFtp", :parent => :file_depot) { uri { "ftp://somehost/export" } } - factory(:file_depot_swift, :class => "FileDepotSwift", :parent => :file_depot) { uri { "swift://swifthost/swiftbucket" } } - factory :file_depot_ftp_with_authentication, :parent => :file_depot_ftp do - after(:create) { |x| x.authentications << FactoryBot.create(:authentication) } - end - factory :file_depot_swift_with_authentication, :parent => :file_depot_swift do - after(:create) { |x| x.authentications << FactoryBot.create(:authentication) } - end -end diff --git a/spec/factories/log_file.rb b/spec/factories/log_file.rb deleted file mode 100644 index 83089e8bc5f..00000000000 --- a/spec/factories/log_file.rb +++ /dev/null @@ -1,7 +0,0 @@ -FactoryBot.define do - factory :log_file do - state { "collecting" } - historical { true } - description { "Default logfile" } - end -end diff --git a/spec/models/database_backup_spec.rb b/spec/models/database_backup_spec.rb deleted file mode 100644 index 7e39de75eb6..00000000000 --- a/spec/models/database_backup_spec.rb +++ /dev/null @@ -1,47 +0,0 @@ -RSpec.describe DatabaseBackup do - context "region" do - let!(:region) { FactoryBot.create(:miq_region, :region => 3) } - - before do - allow(described_class).to receive_messages(:my_region_number => region.region) - end - - it "should set region_name based on my_region_number if database backup has a region" do - backup = FactoryBot.create(:database_backup) - expect(backup.region_name).to eq("region_#{region.region}") - end - - it "should set region_name to region_0 if region is unknown" do - # my_region_number => 0 if REGION file is missing or empty - allow(described_class).to receive_messages(:my_region_number => 0) - backup = FactoryBot.create(:database_backup) - expect(backup.region_name).to eq("region_0") - end - - it "should set class method region_name to MiqRegion.my_region.region" do - expect(DatabaseBackup.region_name).to eq("region_#{region.region}") - end - end - - context "schedule" do - before do - EvmSpecHelper.local_miq_server - - @name = "adhoc schedule" - @sanitized_name = "adhoc_schedule" - @schedule = FactoryBot.create(:miq_schedule, :name => @name) - end - - it "should set schedule_name to schedule_unknown if database backup was not passed a valid schedule id" do - backup = FactoryBot.create(:database_backup) - backup.instance_variable_set(:@schedule, nil) - expect(backup.schedule_name).to eq("schedule_unknown") - end - - it "should set schedule_name to a sanitized version of the schedule's name if database backup was passed a valid schedule id" do - backup = FactoryBot.create(:database_backup) - backup.instance_variable_set(:@sch, @schedule) - expect(backup.schedule_name).to eq(@sanitized_name) - end - end -end diff --git a/spec/models/file_depot_ftp_anonymous_spec.rb b/spec/models/file_depot_ftp_anonymous_spec.rb deleted file mode 100644 index 60b6a8c6b09..00000000000 --- a/spec/models/file_depot_ftp_anonymous_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -RSpec.describe FileDepotFtpAnonymous do - it "should require credentials for anonymous" do - expect(FileDepotFtpAnonymous.requires_credentials?).to eq true - expect(FileDepotFtpAnonymous.new.login_credentials[0]).to eq "anonymous" - end -end diff --git a/spec/models/file_depot_ftp_spec.rb b/spec/models/file_depot_ftp_spec.rb deleted file mode 100644 index ea90e9291d7..00000000000 --- a/spec/models/file_depot_ftp_spec.rb +++ /dev/null @@ -1,141 +0,0 @@ -RSpec.describe FileDepotFtp do - before do - _, @miq_server, @zone = EvmSpecHelper.create_guid_miq_server_zone - end - - let(:connection) { double("FtpConnection") } - let(:uri) { "ftp://server.example.com/uploads" } - let(:file_depot_ftp) { FileDepotFtp.new(:uri => uri) } - let(:log_file) { LogFile.new(:resource => @miq_server, :local_file => "/tmp/file.txt") } - let(:ftp_mock) do - Class.new do - attr_reader :pwd, :content - def initialize(content = {}) - @pwd = '/' - @content = content - end - - def chdir(dir) - newpath = (Pathname.new(pwd) + dir).to_s - if local(newpath).kind_of? Hash - @pwd = newpath - end - end - - private - - def local(path) - local = @content - path.split('/').each do |dir| - next if dir.empty? - local = local[dir] - raise Net::FTPPermError, '550 Failed to change directory.' if local.nil? - end - local - end - end - end - - let(:vsftpd_mock) do - Class.new(ftp_mock) do - def nlst(path = '') - l = local(pwd + path) - l.respond_to?(:keys) ? l.keys : [] - rescue - return [] - end - - def mkdir(dir) - l = local(pwd) - l[dir] = {} - end - - def putbinaryfile(local_path, remote_path) - dir, base = Pathname.new(remote_path).split - l = local(dir.to_s) - l[base.to_s] = local_path - end - end - end - - - context "#file_exists?" do - it "true if file exists" do - file_depot_ftp.ftp = double(:nlst => ["somefile"]) - - expect(file_depot_ftp.file_exists?("somefile")).to be_truthy - end - - it "false if ftp raises on missing file" do - file_depot_ftp.ftp = double - expect(file_depot_ftp.ftp).to receive(:nlst).and_raise(Net::FTPPermError) - - expect(file_depot_ftp.file_exists?("somefile")).to be_falsey - end - - it "false if file missing" do - file_depot_ftp.ftp = double(:nlst => []) - - expect(file_depot_ftp.file_exists?("somefile")).to be_falsey - end - end - - context "#upload_file" do - it 'uploads file to vsftpd with existing directory structure' do - vsftpd = vsftpd_mock.new('uploads' => - {"#{@zone.name}_#{@zone.id}" => - {"#{@miq_server.name}_#{@miq_server.id}" => {}}}) - expect(file_depot_ftp).to receive(:connect).and_return(vsftpd) - file_depot_ftp.upload_file(log_file) - expect(vsftpd.content).to eq('uploads' => - {"#{@zone.name}_#{@zone.id}" => - {"#{@miq_server.name}_#{@miq_server.id}" => - {"Current_region_unknown_#{@zone.name}_#{@zone.id}_#{@miq_server.name}_#{@miq_server.id}.txt".gsub(/\s+/, "_") => - log_file.local_file}}}) - end - - it 'uploads file to vsftpd with empty /uploads directory' do - vsftpd = vsftpd_mock.new('uploads' => {}) - expect(file_depot_ftp).to receive(:connect).and_return(vsftpd) - file_depot_ftp.upload_file(log_file) - expect(vsftpd.content).to eq('uploads' => - {"#{@zone.name}_#{@zone.id}" => - {"#{@miq_server.name}_#{@miq_server.id}" => - {"Current_region_unknown_#{@zone.name}_#{@zone.id}_#{@miq_server.name}_#{@miq_server.id}.txt".gsub(/\s+/, "_") => - log_file.local_file}}}) - end - - it "already exists" do - expect(file_depot_ftp).to receive(:connect).and_return(connection) - expect(file_depot_ftp).to receive(:file_exists?).and_return(true) - expect(connection).not_to receive(:mkdir) - expect(connection).not_to receive(:putbinaryfile) - expect(log_file).not_to receive(:post_upload_tasks) - expect(connection).to receive(:close) - - file_depot_ftp.upload_file(log_file) - end - end - - context "#base_path" do - it "escaped characters" do - file_depot_ftp.uri = "ftp://ftpserver.com/path/abc 123 %^{}|\"<>\\.csv" - expect(file_depot_ftp.send(:base_path).to_s).to eq "path/abc%20123%20%25%5E%7B%7D%7C%22%3C%3E%5C.csv" - end - - it "not escaped characters" do - file_depot_ftp.uri = "ftp://ftpserver.com/path/abc&$!@*()_+:-=;',./.csv" - expect(file_depot_ftp.send(:base_path).to_s).to eq "path/abc&$!@*()_+:-=;',./.csv" - end - end - - context "#merged_uri" do - before do - file_depot_ftp.uri = uri - end - - it "should ignore the uri attribute from the file depot object and return the parameter" do - expect(file_depot_ftp.merged_uri(nil, nil)).to eq nil - end - end -end diff --git a/spec/models/file_depot_nfs_spec.rb b/spec/models/file_depot_nfs_spec.rb deleted file mode 100644 index 3d96058794d..00000000000 --- a/spec/models/file_depot_nfs_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -RSpec.describe FileDepotNfs do - let(:uri) { "nfs://ignore.com/directory" } - let(:actual_uri) { "nfs://actual_bucket/doo_directory" } - let(:file_depot_nfs) { FileDepotNfs.new(:uri => uri) } - - it "should return a valid prefix" do - expect(FileDepotNfs.uri_prefix).to eq "nfs" - end - - describe "#merged_uri" do - before do - file_depot_nfs.uri = uri - end - - it "should ignore the uri set on the depot object and return the uri parameter" do - expect(file_depot_nfs.merged_uri(actual_uri, nil)).to eq actual_uri - end - end -end diff --git a/spec/models/file_depot_swift_spec.rb b/spec/models/file_depot_swift_spec.rb deleted file mode 100644 index 501313f5a7e..00000000000 --- a/spec/models/file_depot_swift_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -RSpec.describe FileDepotSwift do - let(:uri) { "swift://server.example.com/bucket" } - let(:merged_uri) { "swift://server.example.com:5678/bucket?region=test_openstack_region&api_version=v3&domain_id=default" } - let(:merged_default_uri) { "swift://server.example.com:5000/bucket?region=test_openstack_region&api_version=v3&domain_id=default" } - let(:file_depot_swift) { FileDepotSwift.new(:uri => uri) } - it "should require credentials" do - expect(FileDepotSwift.requires_credentials?).to eq true - end - - it "should return a valid prefix" do - expect(FileDepotSwift.uri_prefix).to eq "swift" - end - - describe "#merged_uri" do - before do - file_depot_swift.openstack_region = "test_openstack_region" - file_depot_swift.keystone_api_version = "v3" - file_depot_swift.v3_domain_ident = "default" - end - - it "should return a merged uri with query strings given an empty port" do - expect(file_depot_swift.merged_uri(uri, nil)).to eq merged_default_uri - end - - it "should return a merged uri with query strings when given a valid port" do - expect(file_depot_swift.merged_uri(uri, "5678")).to eq merged_uri - end - end -end diff --git a/spec/models/log_collection_spec.rb b/spec/models/log_collection_spec.rb deleted file mode 100644 index a97c00ffeb2..00000000000 --- a/spec/models/log_collection_spec.rb +++ /dev/null @@ -1,127 +0,0 @@ -RSpec.describe "LogCollection" do - before do - _, @miq_server, @zone = EvmSpecHelper.create_guid_miq_server_zone - end - - context "active log_collection" do - before do - @log_file = FactoryBot.create(:log_file, :state => "collecting") - @miq_server.log_files << @log_file - @task = FactoryBot.create(:miq_task, - :miq_server_id => @miq_server.id, - :name => "Zipped log retrieval for #{@miq_server.name}" - ) - end - - it { expect(@miq_server).to be_log_collection_active } - it { expect(@zone).to be_log_collection_active } - it { expect(@miq_server).to be_log_collection_active_recently } - it { expect(@zone).to be_log_collection_active_recently } - - context "21 minutes ago" do - it { expect(@miq_server.log_collection_active_recently?(21.minutes.ago.utc)).to be_truthy } - it { expect(@zone.log_collection_active_recently?(21.minutes.ago.utc)).to be_truthy } - end - - context "jumping ahead in time 20 minutes" do - before do - Timecop.travel 20.minutes - end - - it { expect(@miq_server).to be_log_collection_active } - it { expect(@zone).to be_log_collection_active } - it { expect(@miq_server).to_not be_log_collection_active_recently } - it { expect(@zone).to_not be_log_collection_active_recently } - - after do - Timecop.return - end - end - end - - context "with a log file instance" do - before do - @log_file = FactoryBot.create(:log_file, :resource => @miq_server) - @region = FactoryBot.create(:miq_region) - @timestamp = "2010" - @fname = "/test.zip" - allow_any_instance_of(LogFile).to receive(:format_log_time).and_return(@timestamp) - end - - context "with a nil region column" do - before do - @region.update(:region => nil) - end - - it "using a historical log file should raise no errors with a nil region column" do - @log_file.update(:historical => true) - expect { @log_file.relative_path_for_upload("/test.zip") }.to_not raise_error - end - end - - context "with my_region nil" do - before do - allow(MiqRegion).to receive(:my_region).and_return(nil) - end - - it "using a historical log file should raise no errors with a nil region association" do - @log_file.update(:historical => true) - expect { @log_file.relative_path_for_upload("/test.zip") }.to_not raise_error - end - end - - %w(models dialogs current archive).each do |log_type| - context "using a #{log_type} log file" do - before do - @fname = "#{log_type}.zip" - allow(MiqRegion).to receive(:my_region).and_return(@region) - @log_file.update(:historical => log_type != "current") - end - - it "should build a destination directory path based on zone and server" do - res = @log_file.relative_path_for_upload(@fname) - expected = "/#{@zone.name}_#{@zone.id}/#{@miq_server.name}_#{@miq_server.id}" - expect(File.dirname(res)).to eq(expected) - end - - it "should build a destination filename based on archive, region, zone, server and date" do - res = @log_file.relative_path_for_upload(@fname) - expected = "#{log_type.capitalize}_region_#{@region.region}_#{@zone.name}_#{@zone.id}_#{@miq_server.name}_#{@miq_server.id}_#{@timestamp}_#{@timestamp}#{File.extname(@fname)}" - expect(File.basename(res)).to eq(expected) - end - end - end - end - - context "queue item and task item creation are atomic" do - context "with error creating task" do - before do - allow(MiqTask).to receive(:new).and_raise("some error message") - LogFile.logs_from_server(@miq_server.id) rescue nil - end - - include_examples("Log Collection should create 0 tasks and 0 queue items") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with error creating queue message" do - before do - allow(MiqQueue).to receive(:put).and_raise("some error message") - LogFile.logs_from_server(@miq_server.id) rescue nil - end - - include_examples("Log Collection should create 0 tasks and 0 queue items") - it { expect(@miq_server).to_not be_log_collection_active } - end - end - - context "Log Collection #synchronize_logs" do - before do - depot = FactoryBot.create(:file_depot) - @zone.update(:log_file_depot_id => depot.id) - end - - include_examples("Log Collection #synchronize_logs", "miq_server") - include_examples("Log Collection #synchronize_logs", "zone") - end -end diff --git a/spec/models/log_file_spec.rb b/spec/models/log_file_spec.rb deleted file mode 100644 index e1c9c44a9f3..00000000000 --- a/spec/models/log_file_spec.rb +++ /dev/null @@ -1,253 +0,0 @@ -RSpec.describe LogFile do - context "With server and zone" do - before do - _, @miq_server, @zone = EvmSpecHelper.create_guid_miq_server_zone - @miq_server.create_log_file_depot!(:type => "FileDepotFtpAnonymous", :uri => "test") - @miq_server.save - end - - it "logs_from_server will queue _request_logs with correct args" do - task_id = described_class.logs_from_server("admin", @miq_server, :current_only => true) - message = MiqQueue.where(:class_name => "LogFile", :method_name => "_request_logs").first - expect(message.priority).to eq(MiqQueue::HIGH_PRIORITY) - expect(message.args).to eq([{:taskid => task_id, :klass => @miq_server.class.name, :id => @miq_server.id, :current_only => true}]) - expect(message.miq_callback).to eq(:instance_id => task_id, :class_name => 'MiqTask', :method_name => :queue_callback_on_exceptions, :args => ['Finished']) - end - - context "logs_from_server with our server" do - before do - @task_id = described_class.logs_from_server("admin", @miq_server) - @tasks = MiqTask.all - @task = @tasks.first - @messages = MiqQueue.where(:class_name => "LogFile", :method_name => "_request_logs") - @message = @messages.first - end - - include_examples("Log Collection should create 1 task and 1 queue item") - - context "with a queued item not picked up, calling delete_active_log_collections" do - # TODO: The current code will never clean up the message or task - before do - @miq_server.reload.delete_active_log_collections - end - - include_examples("Log Collection should create 0 tasks and 0 queue items") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with a queued item not picked up, calling delete_active_log_collections_queue twice" do - let(:messages) { MiqQueue.where(:class_name => "MiqServer", :method_name => "delete_active_log_collections") } - let(:message) { messages.first } - before do - @miq_server.reload.delete_active_log_collections_queue - @miq_server.delete_active_log_collections_queue - end - - it "should create one queued delete" do - expect(messages.length).to eq(1) - end - - it "should have the correct attributes" do - expect(message.instance_id).to eq(@miq_server.id) - expect(message.server_guid).to eq(@miq_server.guid) - expect(message.priority).to eq(MiqQueue::HIGH_PRIORITY) - end - - context "processing the delete queue message" do - before do - message.delivered(*message.deliver) - end - - include_examples("Log Collection should create 0 tasks and 0 queue items") - it { expect(@miq_server).to_not be_log_collection_active } - end - end - - context "with _request_logs queue message raising exception due to stopped server" do - before do - allow_any_instance_of(MiqServer).to receive_messages(:status => "stopped") - @message.delivered(*@message.deliver) - end - - # How does a stopped server dequeue something??? - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - it "delivering MiqServer._request_logs message should call _post_my_logs with correct args" do - expected_options = {:timeout => described_class::LOG_REQUEST_TIMEOUT, - :taskid => @task.id, :miq_task_id => nil, - :callback => {:instance_id => @task.id, :class_name => 'MiqTask', - :method_name => :queue_callback_on_exceptions, :args => ['Finished']}} - expect_any_instance_of(MiqServer).to receive(:_post_my_logs).with(expected_options) - - @message.delivered(*@message.deliver) - end - - context "with MiqServer._request_logs calling _post_my_logs to enqueue" do - let(:message) { MiqQueue.where(:class_name => "MiqServer", :method_name => "post_logs", :instance_id => @miq_server.id, :server_guid => @miq_server.guid, :zone => @miq_server.my_zone).first } - before do - @message.delivered(*@message.deliver) - end - - it "MiqServer#post_logs message should have correct args" do - expect(message.args).to eq([{:taskid => @task.id, :miq_task_id => nil}]) - expect(message.priority).to eq(MiqQueue::HIGH_PRIORITY) - expect(message.miq_callback).to eq(:instance_id => @task.id, :class_name => 'MiqTask', :method_name => :queue_callback_on_exceptions, :args => ['Finished']) - expect(message.msg_timeout).to eq(described_class::LOG_REQUEST_TIMEOUT) - end - - context "with post_logs message" do - it "#post_logs will only post current logs if flag enabled" do - message.args.first[:only_current] = true - expect_any_instance_of(MiqServer).to receive(:post_historical_logs).never - expect_any_instance_of(MiqServer).to receive(:post_current_logs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_dialogs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_models).once - message.delivered(*message.deliver) - end - - it "#post_logs will post both historical and current logs if flag nil" do - expect_any_instance_of(MiqServer).to receive(:post_historical_logs).once - expect_any_instance_of(MiqServer).to receive(:post_current_logs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_dialogs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_models).once - message.delivered(*message.deliver) - end - - it "#post_logs will post both historical and current logs if flag false" do - message.args.first[:only_current] = false - expect_any_instance_of(MiqServer).to receive(:post_historical_logs).once - expect_any_instance_of(MiqServer).to receive(:post_current_logs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_dialogs).once - expect_any_instance_of(MiqServer).to receive(:post_automate_models).once - message.delivered(*message.deliver) - end - end - end - - context "with MiqServer _post_my_logs raising exception" do - before do - allow_any_instance_of(MiqServer).to receive(:_post_my_logs).and_raise - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer _post_my_logs raising timeout exception" do - before do - allow_any_instance_of(MiqServer).to receive(:_post_my_logs).and_raise(Timeout::Error) - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer.post_logs raising missing log_depot settings exception" do - before do - allow_any_instance_of(MiqServer).to receive(:log_depot).and_return(nil) - @message.delivered(*@message.deliver) - - @message = MiqQueue.where(:class_name => "MiqServer", :method_name => "post_logs").first - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer.post_logs fully stubbed" do - before do - @message.delivered(*@message.deliver) - @message = MiqQueue.where(:class_name => "MiqServer", :method_name => "post_logs").first - allow(VMDB::Util).to receive(:zip_logs).and_return('/tmp/blah') - allow(VMDB::Util).to receive(:get_evm_log_for_date).and_return('/tmp/blah') - allow(VMDB::Util).to receive(:get_log_start_end_times).and_return((Time.now.utc - 600), Time.now.utc) - end - - context "with VMDB::Util.zip_logs raising exception" do - before do - allow(VMDB::Util).to receive(:zip_logs).and_raise("some error message") - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with VMDB::Util.zip_logs raising timeout exception" do - before do - allow(VMDB::Util).to receive(:zip_logs).and_raise(Timeout::Error) - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer delete_old_requested_logs raising timeout exception" do - before do - allow_any_instance_of(MiqServer).to receive(:delete_old_requested_logs).and_raise(Timeout::Error) - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer delete_old_requested_logs raising exception" do - before do - allow_any_instance_of(MiqServer).to receive(:delete_old_requested_logs).and_raise("some error message") - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer base_zip_log_name raising exception" do - before do - allow_any_instance_of(MiqServer).to receive(:base_zip_log_name).and_raise("some error message") - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer base_zip_log_name raising timeout exception" do - before do - allow_any_instance_of(MiqServer).to receive(:base_zip_log_name).and_raise(Timeout::Error) - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer format_log_time raising exception" do - before do - allow_any_instance_of(MiqServer).to receive(:format_log_time).and_raise("some error message") - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - - context "with MiqServer format_log_time raising timeout exception" do - before do - allow_any_instance_of(MiqServer).to receive(:format_log_time).and_raise(Timeout::Error) - @message.delivered(*@message.deliver) - end - - include_examples("Log Collection should error out task and queue item") - it { expect(@miq_server).to_not be_log_collection_active } - end - end - end - end -end diff --git a/spec/models/miq_schedule_filter_spec.rb b/spec/models/miq_schedule_filter_spec.rb index ae4931d5d5c..9cb2fcd57b7 100644 --- a/spec/models/miq_schedule_filter_spec.rb +++ b/spec/models/miq_schedule_filter_spec.rb @@ -31,12 +31,6 @@ :sched_action => {:method => "vm_scan"}, :miq_search_id => @search.id ) - - # DB Baskup Schedule - @db_backup = FactoryBot.create(:miq_schedule, - :resource_type => "DatabaseBackup", - :sched_action => {:method => "db_backup"} - ) end context "for a scheduled report" do @@ -88,11 +82,5 @@ expect(targets.length).to eq(1) expect(targets.first.name).to eq("Test VM 2") end - - it "should get the the DatabaseBackup class for a scheduled DB Backup" do - targets = @db_backup.get_targets - expect(targets.length).to eq(1) - expect(targets.first.name).to eq("DatabaseBackup") - end end end diff --git a/spec/models/miq_schedule_spec.rb b/spec/models/miq_schedule_spec.rb index 99f1db0c8d5..50070575522 100644 --- a/spec/models/miq_schedule_spec.rb +++ b/spec/models/miq_schedule_spec.rb @@ -120,62 +120,6 @@ end end - context "DatabaseBackup" do - let(:file_depot) { FactoryBot.create(:file_depot_ftp_with_authentication) } - - before do - @valid_run_ats = [{:start_time => "2010-07-08 04:10:00 Z", :interval => {:unit => "daily", :value => "1"}}, - {:start_time => "2010-07-08 04:10:00 Z", :interval => {:unit => "once"}}] - @valid_schedules = [] - @valid_run_ats.each do |run_at| - @valid_schedules << FactoryBot.create(:miq_schedule_validation, :run_at => run_at, :file_depot => file_depot, :sched_action => {:method => "db_backup"}, :resource_type => "DatabaseBackup") - end - @schedule = @valid_schedules.first - end - - it "exports to array" do - miq_schedule_array = MiqSchedule.export_to_array([@schedule.id], MiqSchedule).first["MiqSchedule"] - expect(miq_schedule_array.slice(*MiqSchedule::ImportExport::SKIPPED_ATTRIBUTES)).to be_empty - expect(miq_schedule_array['file_depot_id']).to eq(file_depot.id) - expect(miq_schedule_array['FileDepotContent']).not_to be(nil) - expect(miq_schedule_array['resource_type']).to eq("DatabaseBackup") - end - - it "imports schedule" do - miq_schedule_array = MiqSchedule.export_to_array([@schedule.id], MiqSchedule).first["MiqSchedule"] - imported_schedule = MiqSchedule.import_from_hash(miq_schedule_array).first - - expect(imported_schedule.sched_action[:method]).to eq("db_backup") - expect(imported_schedule.userid).to eq("system") - expect(imported_schedule.file_depot_id).to eq(file_depot.id) - expect(imported_schedule.resource_type).to eq("DatabaseBackup") - expect(FileDepot.count).to eq(1) - end - end - - context "AutomationTask" do - let(:miq_automate_schedule) do - FactoryBot.create(:miq_automate_schedule, :updated_at => 1.year.ago, :filter => miq_expression, :sched_action => sched_action, :userid => user.userid, :last_run_on => Time.zone.now) - end - - it "exports to array" do - miq_schedule_array = MiqSchedule.export_to_array([miq_automate_schedule.id], MiqSchedule).first["MiqSchedule"] - expect(miq_schedule_array.slice(*MiqSchedule::ImportExport::SKIPPED_ATTRIBUTES)).to be_empty - expect(miq_schedule_array['sched_action'][:options].keys).not_to include(:miq_group_description) - expect(miq_schedule_array['filter']).to be_kind_of(MiqExpression) - expect(miq_schedule_array['resource_type']).to eq("AutomationRequest") - end - - it "imports schedule" do - miq_schedule_array = MiqSchedule.export_to_array([miq_automate_schedule.id], MiqSchedule).first["MiqSchedule"] - imported_schedule = MiqSchedule.import_from_hash(miq_schedule_array).first - - expect(imported_schedule.sched_action[:method]).to eq("run_report") - expect(imported_schedule.userid).to eq(user.userid) - expect(imported_schedule.resource_type).to eq("AutomationRequest") - end - end - context "with resource (ServiceTemplate)" do let(:sched_action) { { :method => "vm_scan", :options => options } } let(:template) { FactoryBot.create(:service_template) } @@ -713,179 +657,6 @@ expect(automate_sched.filter[:parameters].keys).to include("ExtManagementSystem::ext_management_system") end end - - context "valid schedules for db_backup" do - let(:file_depot) { FactoryBot.create(:file_depot_ftp_with_authentication) } - before do - @valid_schedules = [] - @valid_run_ats.each do |run_at| - @valid_schedules << FactoryBot.create(:miq_schedule_validation, :run_at => run_at, :file_depot => file_depot, :sched_action => {:method => "db_backup"}, :resource_type => "DatabaseBackup") - end - @schedule = @valid_schedules.first - end - - context "calling run adhoc_db_backup" do - before do - @task_id = @schedule.run_adhoc_db_backup - @backup_message = MiqQueue.where(:class_name => "DatabaseBackup", :method_name => "backup", :role => "database_operations").first - - @region = FactoryBot.create(:miq_region) - allow(MiqRegion).to receive(:my_region).and_return(@region) - end - - it "should create no database backups" do - expect(DatabaseBackup.count).to eq(0) - expect(@region.database_backups.count).to eq(0) - end - - it "should create 1 miq task" do - tasks = MiqTask.where(:name => "Database backup", :userid => @schedule.userid) - expect(tasks.length).to eq(1) - expect(tasks.first.id).to eq(@task_id) - end - - it "should create one backup queue message for our db backup instance for the database role" do - expect(MiqQueue.where(:class_name => "DatabaseBackup", :method_name => "backup", :role => "database_operations").count).to eq(1) - end - - it "sets backup tasks's timeout to ::Settings.task.active_task_timeout" do - expect(@backup_message.msg_timeout).to eq ::Settings.task.active_task_timeout.to_i_with_method - end - end - - context "calling queue scheduled work via a db_backup schedule firing" do - before do - MiqSchedule.queue_scheduled_work(@schedule.id, nil, Time.now.utc.to_i, nil) - @invoke_actions_message = MiqQueue.where(:class_name => "MiqSchedule", :instance_id => @schedule.id, :method_name => "invoke_actions").first - end - - it "should create an invoke_actions queue message" do - expect(@invoke_actions_message).not_to be_nil - end - - it "should have no other DatabaseBackups" do - expect(DatabaseBackup.count).to eq(0) - end - - context "deliver invoke actions message" do - before do - status, message, result = @invoke_actions_message.deliver - @invoke_actions_message.delivered(status, message, result) - @backup_message = MiqQueue.where(:class_name => "DatabaseBackup", :method_name => "backup", :role => "database_operations").first - - @region = FactoryBot.create(:miq_region) - allow(MiqRegion).to receive(:my_region).and_return(@region) - end - - it "should create no database backups" do - expect(DatabaseBackup.count).to eq(0) - expect(@region.database_backups.count).to eq(0) - end - - it "should create 1 miq task" do - expect(MiqTask.where(:name => "Database backup", :userid => @schedule.userid).count).to eq(1) - end - - it "should create one backup queue message for our db backup instance for the database role" do - expect(MiqQueue.where(:class_name => "DatabaseBackup", :method_name => "backup", :role => "database_operations").count).to eq(1) - end - - context "_backup is stubbed" do - before do - # stub out the actual backup behavior - allow_any_instance_of(DatabaseBackup).to receive(:_backup) - end - - context "deliver DatabaseBackup.backup message" do - before do - @status, message, result = @backup_message.deliver - @backup_message.delivered(@status, message, result) - end - - it "should create 1 database backup, queue message is ok, and task is Ok and Finished" do - expect(@status).to eq("ok") - expect(@region.database_backups.count).to eq(1) - expect(MiqTask.where(:state => "Finished", :status => "Ok").count).to eq(1) - end - end - - context "deliver DatabaseBackup.backup message with adhoc true" do - before do - @schedule.update_attribute(:adhoc, true) - @schedule_id = @schedule.id - @status, message, result = @backup_message.deliver - @backup_message.delivered(@status, message, result) - end - - it "should delete the adhoc schedule" do - expect(MiqSchedule.exists?(@schedule_id)).not_to be_truthy - end - end - - context "deliver DatabaseBackup.backup message with adhoc false" do - before do - @schedule.update_attribute(:adhoc, false) - @schedule_id = @schedule.id - @status, message, result = @backup_message.deliver - @backup_message.delivered(@status, message, result) - end - - it "should not delete the schedule" do - expect(MiqSchedule.exists?(@schedule_id)).to be_truthy - end - end - end - end - end - - it "should have valid depots" do - @valid_schedules.each { |sch| expect(sch.valid?).to be_truthy } - end - - it "should return the expected FileDepot subclass" do - @valid_schedules.each { |sch| expect(sch.file_depot).to be_kind_of(FileDepotFtp) } - end - end - end - - context "#verify_file_depot" do - let(:params) { {:uri_prefix => "ftp", :uri => "ftp://ftp.example.com", :name => "Test Backup Depot", :username => "user", :password => "password"} } - - it "builds a file_depot of the correct type and validates it, does not save" do - expect_any_instance_of(FileDepotFtp).to receive(:verify_credentials).and_return(true) - - MiqSchedule.new.verify_file_depot(params) - - expect(Authentication.count).to eq(0) - expect(FileDepot.count).to eq(0) - expect(MiqSchedule.count).to eq(0) - end - - it "saves the file_depot when params[:save] is passed in, does not validate" do - MiqSchedule.new.verify_file_depot(params.merge(:save => true)) - - expect(Authentication.count).to eq(1) - expect(FileDepot.count).to eq(1) - expect(MiqSchedule.count).to eq(0) - end - - let(:swift_file_depot) { FactoryBot.create(:file_depot_swift_with_authentication) } - let(:swift_params) { {:uri_prefix => "swift", :uri => "swift://swift.example.com/test_depot", :name => "Test Backup Swift Depot", :username => "user", :password => "password"} } - let(:swift_run_at) { {:interval => {:value => "1", :unit => "hourly"}, :start_time => "2012-03-10 01:35:00 Z", :tz => "Central Time (US & Canada)"} } - - it "does not merge the swift uri if the port is blank" do - swift_schedule = FactoryBot.create(:miq_schedule_validation, :run_at => swift_run_at, :file_depot => swift_file_depot, :sched_action => {:method => "db_backup"}, :resource_type => "DatabaseBackup") - swift_schedule.verify_file_depot(swift_params.merge(:save => true)) - expect(swift_schedule.file_depot.uri).to eq(swift_params[:uri]) - end - - it "merges the swift uri and port if the port is specified" do - another_swift_schedule = FactoryBot.create(:miq_schedule_validation, :run_at => swift_run_at, :file_depot => swift_file_depot, :sched_action => {:method => "db_backup"}, :resource_type => "DatabaseBackup") - swift_api_port = 5000 - merged_uri = "swift://swift.example.com:5000/test_depot" - another_swift_schedule.verify_file_depot(swift_params.merge(:swift_api_port => swift_api_port, :save => true)) - expect(another_swift_schedule.file_depot.uri).to eq(merged_uri) - end end describe ".updated_since" do diff --git a/spec/models/miq_server/log_management_spec.rb b/spec/models/miq_server/log_management_spec.rb deleted file mode 100644 index 1768925454a..00000000000 --- a/spec/models/miq_server/log_management_spec.rb +++ /dev/null @@ -1,281 +0,0 @@ -def stub_vmdb_util_methods_for_collection_log - allow(VMDB::Util).to receive(:zip_logs) - allow(VMDB::Util).to receive(:compressed_log_patterns).and_return(["log/evm.1122.gz"]) - allow(VMDB::Util).to receive(:get_evm_log_for_date).and_return("20151209_141429_20151217_140845") - allow(VMDB::Util).to receive(:get_log_start_end_times).and_return([Time.zone.now, Time.zone.now]) -end - -shared_examples "post_[type_of_log]_logs" do |type, type_of_log| - it "uses #{type} file_depot in LogFile for upload" do - @zone.log_file_depot = zone_depot - @miq_server.log_file_depot = server_depot - - stub_vmdb_util_methods_for_collection_log - allow_any_instance_of(LogFile).to receive(:upload) - allow_any_instance_of(MiqServer).to receive(:current_log_patterns) - - method = "post_#{type_of_log}_logs".to_sym - @miq_server.send(method, miq_task.id, @miq_server.log_depot(type)) - log_file_depot = LogFile.first.file_depot - - if type == "Zone" - expect(log_file_depot).to eq(zone_depot) - expect(log_file_depot).not_to eq(server_depot) - else - expect(log_file_depot).not_to eq(zone_depot) - expect(log_file_depot).to eq(server_depot) - end - - expect(LogFile.first.miq_task).to eq(miq_task) - end -end - -shared_examples "post_logs_uses_depot" do |is_zone_depot, is_server_depot, context| - it "uses #{context} depot" do - if is_zone_depot - @zone.log_file_depot = zone_depot - end - - if is_server_depot - @miq_server.log_file_depot = server_depot - end - - stub_vmdb_util_methods_for_collection_log - allow_any_instance_of(LogFile).to receive(:upload) - allow_any_instance_of(MiqServer).to receive(:current_log_patterns) - - @miq_server.post_logs(:taskid => miq_task.id, :context => context) - - log_file_depot = LogFile.first.file_depot - - if context == "Zone" - expect(log_file_depot).to eq(zone_depot) - else - expect(log_file_depot).to eq(server_depot) - end - end -end - -shared_examples "post_logs_fails" do |is_zone_depot, is_server_depot, context| - it "raises error 'Log depot settings not configured'" do - if is_zone_depot - @zone.log_file_depot = zone_depot - end - - if is_server_depot - @miq_server.log_file_depot = server_depot - end - - stub_vmdb_util_methods_for_collection_log - allow_any_instance_of(LogFile).to receive(:upload) - allow_any_instance_of(MiqServer).to receive(:current_log_patterns) - - expect do - @miq_server.post_logs(:taskid => miq_task.id, :context => context) - end.to raise_error(RuntimeError, "Log depot settings not configured") - end -end - -RSpec.describe MiqServer do - context "LogManagement" do - let(:server_depot) { FactoryBot.create(:file_depot) } - let(:zone_depot) { FactoryBot.create(:file_depot) } - let(:miq_task) { FactoryBot.create(:miq_task) } - - before do - _, @miq_server, @zone = EvmSpecHelper.create_guid_miq_server_zone - @miq_server2 = FactoryBot.create(:miq_server, :zone => @zone) - end - - context "#pg_data_log_patterns" do - it "nil pg_data_dir" do - allow(@miq_server).to receive_messages(:pg_data_dir => nil) - expect(@miq_server.pg_log_patterns).to eql [] - end - - it "pg_data_dir set" do - allow(@miq_server).to receive_messages(:pg_data_dir => '/var/lib/pgsql/data') - expected = %w(/var/lib/pgsql/data/*.conf /var/lib/pgsql/data/pg_log/* /etc/manageiq/postgresql.conf.d/*) - expect(@miq_server.pg_log_patterns.collect(&:to_s)).to match_array expected - end - end - - it "#current_log_patterns" do - stub_settings(:log => {:collection => {:current => {:pattern => %w(/var/log/syslog*)}}}) - allow(@miq_server).to receive_messages(:pg_log_patterns => %w(/var/lib/pgsql/data/*.conf)) - expect(@miq_server.current_log_patterns).to match_array %w(/var/log/syslog* /var/lib/pgsql/data/*.conf) - end - - it "#current_log_patterns with pg_logs duplicated in current_log_pattern_configuration" do - stub_settings( - :log => {:collection => {:current => {:pattern => %w(/var/log/syslog* /var/lib/pgsql/data/*.conf)}}}) - allow(@miq_server).to receive_messages(:pg_log_patterns => %w(/var/lib/pgsql/data/*.conf)) - expect(@miq_server.current_log_patterns).to match_array %w(/var/log/syslog* /var/lib/pgsql/data/*.conf) - end - - context "post current/historical/models/dialogs" do - let(:task) { FactoryBot.create(:miq_task) } - let(:compressed_log_patterns) { [Rails.root.join("log", "evm*.log.gz").to_s] } - let(:current_log_patterns) { [Rails.root.join("log", "evm.log").to_s] } - let(:compressed_evm_log) { Rails.root.join("evm.log-20180319.gz").to_s } - let(:log_start) { Time.zone.parse("2018-05-11 11:33:12 UTC") } - let(:log_end) { Time.zone.parse("2018-05-11 15:34:16 UTC") } - let(:daily_log) { Rails.root.join("data", "user", "system", "evm_server_daily.zip").to_s } - let(:log_depot) { FactoryBot.create(:file_depot) } - let!(:region) { MiqRegion.seed } - let(:zone) { @miq_server.zone } - before do - require 'vmdb/util' - allow(VMDB::Util).to receive(:compressed_log_patterns).and_return(compressed_log_patterns) - allow(VMDB::Util).to receive(:get_evm_log_for_date).and_return(compressed_evm_log) - allow(VMDB::Util).to receive(:get_log_start_end_times).and_return([log_start, log_end]) - allow(VMDB::Util).to receive(:zip_logs).and_return(daily_log) - allow(@miq_server).to receive(:current_log_patterns).and_return(current_log_patterns) - allow(@miq_server).to receive(:backup_automate_dialogs) - allow(@miq_server).to receive(:backup_automate_models) - %w(historical_logfile current_logfile).each do |kind| - logfile = FactoryBot.create(:log_file, :historical => kind == "historical_logfile") - allow(logfile).to receive(:upload) - allow(LogFile).to receive(kind).and_return(logfile) - end - end - - %w( - Archive post_historical_logs - Current post_current_logs - Models post_automate_models - Dialogs post_automate_dialogs - ).each_slice(2) do |name, method| - it "##{method}" do - logfile = nil - - now = Time.zone.now - Timecop.freeze(now) do - @miq_server.send(method, task.id, log_depot) - logfile = @miq_server.reload.log_files.first - end - - if %w[Current Archive].include?(name) - expected_name = [name, "region", region.region, zone.name, zone.id, @miq_server.name, @miq_server.id, "20180511_113312 20180511_153416"].join(" ") - expect(logfile).to have_attributes( - :file_depot => log_depot, - :local_file => daily_log, - :logging_started_on => log_start, - :logging_ended_on => log_end, - :name => expected_name, - :description => "Logs for Zone #{@miq_server.zone.name} Server #{@miq_server.name} 20180511_113312 20180511_153416", - :miq_task_id => task.id - ) - expected_filename = "#{name}_region_#{region.region}_#{zone.name}_#{zone.id}_#{@miq_server.name}_#{@miq_server.id}_20180511_113312_20180511_153416.zip" - expected_filename.gsub!(/\s+/, "_") - expect(logfile.destination_file_name).to eq(expected_filename) - else - formatted_now = now.strftime("%Y%m%d_%H%M%S") - expected_name = [name, "region", region.region, zone.name, zone.id, @miq_server.name, @miq_server.id, formatted_now, formatted_now].join(" ") - expect(logfile).to have_attributes( - :file_depot => log_depot, - :local_file => daily_log, - :logging_started_on => be_within(1).of(now), - :logging_ended_on => be_within(1).of(now), - :name => expected_name, - :description => "Logs for Zone #{@miq_server.zone.name} Server #{@miq_server.name} #{formatted_now} #{formatted_now}", - :miq_task_id => task.id - ) - end - expect(task.reload).to have_attributes( - :message => "#{name} log files from #{@miq_server.name} #{@miq_server.zone.name} MiqServer #{@miq_server.id} are posted", - :state => "Active", - :status => "Ok", - ) - end - end - end - - context "#synchronize_logs" do - it "passes along server override" do - @miq_server.synchronize_logs("system", @miq_server2) - expect(MiqTask.first.miq_server_id).to eql @miq_server2.id - expect(MiqQueue.first.args.first[:id]).to eql @miq_server2.id - end - - it "passes 'self' server if no server arg" do - @miq_server2.synchronize_logs("system") - expect(MiqTask.first.miq_server_id).to eql @miq_server2.id - expect(MiqQueue.first.args.first[:id]).to eql @miq_server2.id - end - end - - describe "#log_depot" do - it "server log_file_depot configured" do - @miq_server.log_file_depot = server_depot - expect(@miq_server.log_depot("MiqServer")).to eq(server_depot) - end - - it "zone log_file_depot configured" do - @zone.log_file_depot = zone_depot - expect(@miq_server.log_depot("Zone")).to eq(zone_depot) - end - - it "server and zone log_file_depot configured" do - @miq_server.log_file_depot = server_depot - @zone.log_file_depot = zone_depot - expect(@miq_server.log_depot("Zone")).to eq(zone_depot) - expect(@miq_server.log_depot("MiqServer")).to eq(server_depot) - end - end - - describe "#post_historical_logs" do - context "Server" do - include_examples "post_[type_of_log]_logs", "MiqServer", :historical - end - - context "Zone" do - include_examples "post_[type_of_log]_logs", "Zone", :historical - end - end - - describe "#post_current_logs" do - context "Server" do - include_examples "post_[type_of_log]_logs", "MiqServer", :current - end - - context "Zone" do - include_examples "post_[type_of_log]_logs", "Zone", :current - end - end - - describe "#post_logs" do - context "Zone collection log requested, Zone depot is defined, MiqServer is defined" do - include_examples "post_logs_uses_depot", true, true, "Zone" - end - - context "Zone collection log requested, Zone depot is defined, MiqServer is not defined" do - include_examples "post_logs_uses_depot", true, false, "Zone" - end - - context "Zone collection log requested, zone depot is not defined, MiqServer defined" do - include_examples "post_logs_fails", false, true, "Zone" - end - - context "Zone collection log requested, zone depot is not defined, MiqServer is not defined" do - include_examples "post_logs_fails", false, false, "Zone" - end - - context "MiqServer collection log requested, Zone depot is defined, MiqServer is defined" do - include_examples "post_logs_uses_depot", true, true, "MiqServer" - end - - context "MiqServer collection log requested, Zone depot is defined, MiqServer is not defined" do - include_examples "post_logs_fails", true, false, "MiqServer" - end - - context "MiqServer collection log requested, zone depot is not defined, server defined" do - include_examples "post_logs_uses_depot", false, true, "MiqServer" - end - - context "MiqServer collection log requested, zone depot is not defined, MiqServer is not defined" do - include_examples "post_logs_fails", false, false, "MiqServer" - end - end - end -end diff --git a/spec/models/miq_task/purging_spec.rb b/spec/models/miq_task/purging_spec.rb index 26d61b24f1a..36c6f91354a 100644 --- a/spec/models/miq_task/purging_spec.rb +++ b/spec/models/miq_task/purging_spec.rb @@ -5,14 +5,12 @@ Timecop.freeze(8.days.ago) do @old_task = VmScan.create_job(:guid => "old").miq_task FactoryBot.create(:binary_blob, :name => "old", :resource_type => 'MiqTask', :resource_id => @old_task.id) - FactoryBot.create(:log_file, :name => "old", :miq_task_id => @old_task.id) end Timecop.freeze(6.days.ago) do @new_task = VmScan.create_job(:guid => "recent").miq_task @new_task.state_finished FactoryBot.create(:binary_blob, :name => "recent", :resource_type => 'MiqTask', :resource_id => @new_task.id) - FactoryBot.create(:log_file, :name => "recent", :miq_task_id => @new_task.id) end end @@ -25,8 +23,6 @@ expect(described_class.all).to match_array([@new_task]) expect(BinaryBlob.count).to eq(1) expect(BinaryBlob.first.name).to eq("recent") - expect(LogFile.count).to eq(1) - expect(LogFile.first.name).to eq("recent") expect(Job.count).to eq(1) expect(Job.first.guid).to eq("recent") end diff --git a/spec/support/examples_group/shared_examples_for_log_collection.rb b/spec/support/examples_group/shared_examples_for_log_collection.rb deleted file mode 100644 index a87827b7878..00000000000 --- a/spec/support/examples_group/shared_examples_for_log_collection.rb +++ /dev/null @@ -1,58 +0,0 @@ -shared_examples_for "Log Collection #synchronize_logs" do |type| - let(:instance) { instance_variable_get("@#{type}") } - - it "#{type.camelize} no args" do - expect(LogFile).to receive(:logs_from_server).with(MiqServer.my_server, hash_excluding(:only_current)) - - instance.synchronize_logs - end - - it "#{type.camelize} with options" do - expect(LogFile).to receive(:logs_from_server).with(MiqServer.my_server, hash_including(:only_current => true)) - - instance.synchronize_logs(:only_current => true) - end - - it "#{type.camelize} user with options" do - expect(LogFile).to receive(:logs_from_server).with("test", MiqServer.my_server, hash_including(:only_current => false)) - - instance.synchronize_logs("test", :only_current => false) - end -end - -shared_examples_for "Log Collection should create 1 task and 1 queue item" do - it "should create 1 task" do - expect(@tasks.length).to eq(1) - expect(@task.id).to eq(@task_id) - expect(@task.miq_server_id).to eq(@miq_server.id) - expect(@task.name).to include("Zipped log retrieval") - end - - it "should create 1 queue message" do - expect(@messages.length).to eq(1) - expect(@message.args.first).to eq(:taskid => @task.id, :klass => @miq_server.class.name, :id => @miq_server.id) - end -end - -shared_examples_for "Log Collection should create 0 tasks and 0 queue items" do - it "should create 0 unfinished tasks" do - expect(MiqTask.where("state != ?", "Finished").count).to eq(0) - end - - it "should create 0 queue messages" do - expect(MiqQueue.where("state not in (?)", %w(ok ready error)).count).to eq(0) - end -end - -shared_examples_for "Log Collection should error out task and queue item" do - it "should error out queue item through queue callback" do - expect(["timeout", "error"]).to include(@message.state) - end - - it "should error out task item queue callback" do - @task.reload - - expect(@task.state).to eq("Finished") - expect(["Timeout", "Error"]).to include(@task.status) - end -end From 8535e53fb3dff7b3d129f7fc1ed35c5307f9f341 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Wed, 4 Aug 2021 16:41:14 -0500 Subject: [PATCH 2/2] Remove EvmDatabaseOps and related rake tasks Well... mostly. A single file is needed as the appliance_console uses this method to check the current database connection. This code could be moved into the console itself, but since it was released recently, this allows that code to still function until a new version not requiring it can be released. --- lib/evm_database_ops.rb | 236 --------------------- lib/tasks/evm_dba.rake | 113 +++------- spec/lib/evm_database_ops_spec.rb | 342 ------------------------------ 3 files changed, 28 insertions(+), 663 deletions(-) delete mode 100644 spec/lib/evm_database_ops_spec.rb diff --git a/lib/evm_database_ops.rb b/lib/evm_database_ops.rb index 04928c35fb9..fdb55e24f84 100644 --- a/lib/evm_database_ops.rb +++ b/lib/evm_database_ops.rb @@ -1,243 +1,7 @@ -$LOAD_PATH << File.expand_path(__dir__) -require 'util/postgres_admin' - -require 'mount/miq_generic_mount_session' -require 'util/miq_object_storage' - class EvmDatabaseOps - include Vmdb::Logging - BACKUP_TMP_FILE = "/tmp/miq_backup".freeze - DUMP_TMP_FILE = "/tmp/miq_pg_dump".freeze - - DEFAULT_OPTS = {:dbname => 'vmdb_production'} - - def self.backup_destination_free_space(file_location) - require 'fileutils' - parent_directory = File.dirname(file_location) - FileUtils.mkdir_p(parent_directory) - - free_space = begin - output = AwesomeSpawn.run!("df", :params => {:P => parent_directory}, :combined_output => true).output - data_line = output.split("\n")[1] if output.kind_of?(String) - data_line.split[3].to_i * 1024 if data_line - end - - free_space || 0 - end - - def self.database_size(opts) - PostgresAdmin.database_size(opts) - end - - def self.validate_free_space(database_opts) - free_space = backup_destination_free_space(database_opts[:local_file]) - db_size = database_size(database_opts) - if free_space > db_size - _log.info("[#{database_opts[:dbname]}] with database size: [#{db_size} bytes], free space at [#{database_opts[:local_file]}]: [#{free_space} bytes]") - else - msg = "Destination location: [#{database_opts[:local_file]}], does not have enough free disk space: [#{free_space} bytes] for database of size: [#{db_size} bytes]" - _log.warn(msg) - defined?(::MiqEvent) && MiqEvent.raise_evm_event_queue(MiqServer.my_server, "evm_server_db_backup_low_space", :event_details => msg) - raise MiqException::MiqDatabaseBackupInsufficientSpace, msg - end - end - - def self.backup(db_opts, connect_opts = {}) - # db_opts: - # :dbname => 'vmdb_production', - # :username => 'root', - # :local_file => "/tmp/backup_1", - Backup locally to the file specified - - # connect_opts: - # :uri => "smb://dev005.manageiq.com/share1", - # :username => 'samba_one', - # :password => 'Zug-drep5s', - # :remote_file_name => "backup_1", - Provide a base file name for the uploaded file - - uri = with_file_storage(:backup, db_opts, connect_opts) do |database_opts| - backup_result = PostgresAdmin.backup(database_opts) - backup_result - end - _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been backed up to file: [#{uri}]") - uri - end - - def self.dump(db_opts, connect_opts = {}) - # db_opts and connect_opts similar to .backup - - uri = with_file_storage(:dump, db_opts, connect_opts) do |database_opts| - PostgresAdmin.backup_pg_dump(database_opts) - end - _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been dumped up to file: [#{uri}]") - uri - end - - def self.restore(db_opts, connect_opts = {}) - # db_opts: - # :local_file => "/tmp/backup_1", - Restore from this local file - # :dbname => 'vmdb_production' - # :username => 'root' - - # connect_opts: - # :uri => "smb://dev005.manageiq.com/share1/db_backup/miq_pg_backup_20100719_215444", - # :username => 'samba_one', - # :password => 'Zug-drep5s', - - uri = with_file_storage(:restore, db_opts, connect_opts) do |database_opts, backup_type| - prepare_for_restore(database_opts[:local_file], backup_type) - - # remove all the connections before we restore; AR will reconnect on the next query - ActiveRecord::Base.connection_pool.disconnect! - PostgresAdmin.restore(database_opts.merge(:backup_type => backup_type)) - end - _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been restored from file: [#{uri}]") - uri - end - - private_class_method def self.merged_db_opts(db_opts) - DEFAULT_OPTS.merge(db_opts) - end - - STORAGE_ACTIONS_TO_METHODS = { :backup => :add, :dump => :add, :restore => :download }.freeze - private_class_method def self.with_file_storage(action, db_opts, connect_opts) - db_opts = merged_db_opts(db_opts) - - if db_opts[:local_file].nil? - if action == :restore - uri = connect_opts[:uri] - - connect_uri = URI::Generic.new(*URI.split(uri)) - connect_uri.path = File.dirname(connect_uri.path) - connect_opts[:uri] = connect_uri.to_s - else - connect_opts[:remote_file_name] ||= File.basename(backup_file_name(action)) - # - # If the passed in URI contains query parameters, ignore them - # when creating the dump file name. They'll be used in the session object. - # - uri_parts = [connect_opts[:uri].split('?')[0]] - uri_parts << (action == :dump ? "db_dump" : "db_backup") unless connect_opts[:skip_directory] - uri_parts << connect_opts[:remote_file_name] - uri = File.join(uri_parts) - end - else - uri = db_opts[:local_file] - - # HACK(ish): This just puts the bare minimum necessary for URI.parse to - # recognize the :uri option as "file" scheme, and allows MiqFileStorage - # to then instantiate MiqLocalMountSession below in the - # `.with_interface_class` method. - connect_opts[:uri] = "file://" - end - - MiqFileStorage.with_interface_class(connect_opts) do |file_storage| - send_args = [uri, db_opts[:byte_count]].compact - - if action == :restore - # `MiqFileStorage#download` requires a `nil` passed to the block form - # to accommodate streaming - send_args.unshift(nil) - magic_numbers = { - :pgdump => PostgresAdmin::PG_DUMP_MAGIC, - :basebackup => PostgresAdmin::BASE_BACKUP_MAGIC - } - backup_type = file_storage.magic_number_for(uri, :accepted => magic_numbers) - end - - # Note: `input_path` will always be a fifo stream (input coming from - # PostgresAdmin, and the output going to the `uri`), since we want to - # maintain the same interface for all backup types. - # - # This means that `uri` will always be the final destination, but - # `input_path` below will be an intermediary fifo that will take the - # input from `pg_dump`, `pg_restore`, or `pg_basebackup`, and streams the - # results from those commands (in ruby) it to whatever file storage - # endpoint `uri` is pointing to. - # - # This also makes sure that the streamed output is never written to disk - # locally, unless `uri` is targeting the local machine. This is why we - # set `db_opts` local file to that stream. - file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], *send_args) do |input_path| - db_opts[:local_file] = input_path - if action == :restore - yield(db_opts, backup_type) - else - if file_storage.class <= MiqGenericMountSession - # Only check free space on "mountable" storages - # - # For database dumps, this isn't going to be as accurate (since the - # dump size will probably be larger than the calculated DB size), but - # it still won't hurt to do as a generic way to get a rough idea if - # we have enough disk space on the appliance for the task. - free_space_opts = db_opts.merge(:local_file => file_storage.uri_to_local_path(uri)) - validate_free_space(free_space_opts) - end - yield(db_opts) - end - end - end - - uri - end - - private_class_method def self.prepare_for_restore(filename, backup_type = nil) - backup_type ||= validate_backup_file_type(filename) - - if application_connections? - message = "Database restore failed. Shut down all evmserverd processes before attempting a database restore" - _log.error(message) - raise message - end - - MiqRegion.replication_type = :none - - connection_count = backup_type == :basebackup ? VmdbDatabaseConnection.unscoped.where(:backend_type => "client backend").count : VmdbDatabaseConnection.count - if connection_count > 1 - message = "Database restore failed. #{connection_count - 1} connections remain to the database." - _log.error(message) - raise message - end - end - - private_class_method def self.validate_backup_file_type(filename) - if PostgresAdmin.base_backup_file?(filename) - :basebackup - elsif PostgresAdmin.pg_dump_file?(filename) - :pgdump - else - message = "#{filename} is not in a recognized database backup format" - _log.error(message) - raise message - end - end - - private_class_method def self.application_connections? - VmdbDatabaseConnection.all.map(&:application_name).any? { |app_name| app_name.start_with?("MIQ") } - end - - def self.gc(options = {}) - PostgresAdmin.gc(options) - end - def self.database_connections(database = nil, _type = :all) database ||= ActiveRecord::Base.configurations[Rails.env]["database"] conn = ActiveRecord::Base.connection conn.client_connections.count { |c| c["database"] == database } end - - def self.upload(connect_opts, local_file, destination_file) - MiqGenericMountSession.in_depot_session(connect_opts) { |session| session.upload(local_file, destination_file) } - destination_file - end - - def self.download(connect_opts, local_file) - MiqGenericMountSession.in_depot_session(connect_opts) { |session| session.download(local_file, connect_opts[:uri]) } - local_file - end - - def self.backup_file_name(action = :backup) - time_suffix = Time.now.utc.strftime("%Y%m%d_%H%M%S") - "#{action == :backup ? BACKUP_TMP_FILE : DUMP_TMP_FILE}_#{time_suffix}" - end - private_class_method :backup_file_name end diff --git a/lib/tasks/evm_dba.rake b/lib/tasks/evm_dba.rake index b91fe3e2781..c53adf86e21 100644 --- a/lib/tasks/evm_dba.rake +++ b/lib/tasks/evm_dba.rake @@ -117,8 +117,34 @@ namespace :evm do opt :table, "Tablename to reindex (if only perorm on one)", :type => :string end - opts = opts.delete_if { |_, v| v == false } - EvmDatabaseOps.gc(opts) + args = {} + opts = opts.delete_if { |_, v| v == false } + options = (options[:aggressive] ? GC_AGGRESSIVE_DEFAULTS : GC_DEFAULTS).merge(opts) + + pg_env = {"PGUSER" => options[:username], "PGPASSWORD" => options[:password]}.delete_blanks + + raise "Vacuum requires database" unless options[:dbname] + + vacuum_args = {} + vacuum_args[:analyze] = nil if options[:analyze] + vacuum_args[:full] = nil if options[:full] + vacuum_args[:verbose] = nil if options[:verbose] + vacuum_args[:table] = options[:table] if options[:table] + + run_command("vacuumdb", opts, args) + + $log.info("Running command... #{AwesomeSpawn.build_command_line("vacuumdb", params)}") + output = AwesomeSpawn.run!("vacuumdb", :params => params, :env => pg_env).output + $log.info("Output... #{result}") if output.to_s.length > 0 + + if options[:reindex] + reindex_args = {} + reindex_args[:table] = options[:table] if options[:table] + + $log.info("Running command... #{AwesomeSpawn.build_command_line("reindexdb", params)}") + output = AwesomeSpawn.run!("reindexdb", :params => params, :env => pg_env).output + $log.info("Output... #{result}") if output.to_s.length > 0 + end exit # exit so that parameters to the first rake task are not run as rake tasks end @@ -188,89 +214,6 @@ namespace :evm do exit # exit so that parameters to the first rake task are not run as rake tasks end - - # Example usage: - # bin/rake evm:db:backup:local -- --local-file /tmp/db_backup_test --dbname vmdb_production - # bin/rake evm:db:backup:remote -- --uri smb://dev005.manageiq.com/share1 --uri-username samba_one --uri-password "abc" --remote-file-name region1 - # bin/rake evm:db:restore:local -- --local-file /tmp/db_backup_test - # bin/rake evm:db:restore:remote -- --uri smb://dev005.manageiq.com/share1/db_backup/region1 --uri-username samba_one --uri-password "abc" - - namespace :backup do - require File.expand_path(File.join(Rails.root, "lib", "evm_database_ops")) - desc 'Backup the local ManageIQ EVM Database (VMDB) to a local file' - task :local do - opts = EvmDba.with_options(:local_file, :splitable, :db_credentials) - - EvmDatabaseOps.backup(opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - - desc 'Backup the local ManageIQ EVM Database (VMDB) to a remote file' - task :remote do - opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :splitable, :db_credentials) - - db_opts = EvmDba.collect_db_opts(opts) - connect_opts = EvmDba.collect_connect_opts(opts) - - EvmDatabaseOps.backup(db_opts, connect_opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - end - - namespace :dump do - require Rails.root.join("lib", "evm_database_ops").expand_path.to_s - desc 'Dump the local ManageIQ EVM Database (VMDB) to a local file' - task :local do - opts = EvmDba.with_options(:local_file, :splitable, :db_credentials, :exclude_table_data) - - EvmDatabaseOps.dump(opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - - desc 'Dump the local ManageIQ EVM Database (VMDB) to a remote file' - task :remote do - opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :splitable, :db_credentials, :exclude_table_data) - - db_opts = EvmDba.collect_db_opts(opts) - connect_opts = EvmDba.collect_connect_opts(opts) - - EvmDatabaseOps.dump(db_opts, connect_opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - end - - namespace :restore do - desc 'Restore the local ManageIQ EVM Database (VMDB) from a local backup file' - task :local => :environment do - opts = EvmDba.with_options(:local_file, :db_credentials) - - # If running through runner, disconnect any local connections - ActiveRecord::Base.clear_all_connections! if ActiveRecord && ActiveRecord::Base - - EvmDatabaseOps.restore(opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - - desc 'Restore the local ManageIQ EVM Database (VMDB) from a remote backup file' - task :remote => :environment do - opts = EvmDba.with_options(:remote_uri, :aws, :db_credentials) - - db_opts = EvmDba.collect_db_opts(opts) - connect_opts = EvmDba.collect_connect_opts(opts) - - # If running through runner, disconnect any local connections - ActiveRecord::Base.clear_all_connections! if ActiveRecord && ActiveRecord::Base - - EvmDatabaseOps.restore(db_opts, connect_opts) - - exit # exit so that parameters to the first rake task are not run as rake tasks - end - end end end diff --git a/spec/lib/evm_database_ops_spec.rb b/spec/lib/evm_database_ops_spec.rb deleted file mode 100644 index d43f88650b9..00000000000 --- a/spec/lib/evm_database_ops_spec.rb +++ /dev/null @@ -1,342 +0,0 @@ -RSpec.describe EvmDatabaseOps do - let(:file_storage) { double("MiqSmbSession", :disconnect => nil) } - let(:local_backup) { "/tmp/backup_1" } - let(:input_path) { "foo/bar/mkfifo" } - let(:run_db_ops) { @db_opts.dup.merge(:local_file => input_path) } - let(:tmpdir) { Rails.root.join("tmp") } - - before do - allow(MiqFileStorage).to receive(:with_interface_class).and_yield(file_storage) - end - - context "#backup" do - before do - @connect_opts = {:username => 'blah', :password => 'blahblah', :uri => "smb://myserver.com/share"} - @db_opts = {:username => 'root', :dbname => 'vmdb_production' } - allow(file_storage).to receive(:settings_mount_point).and_return(tmpdir.to_s) - allow(file_storage).to receive(:uri_to_local_path).and_return(tmpdir.join("share").to_s) - allow(file_storage).to receive(:add).and_yield(input_path) - - allow(FileUtils).to receive(:mv).and_return(true) - allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(200.megabytes) - allow(EvmDatabaseOps).to receive(:database_size).and_return(100.megabytes) - - MiqEvent # ensure the MiqEvent class is defined so we can verify raised events - end - - it "locally" do - @db_opts[:local_file] = local_backup - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to eq(local_backup) - end - - it "defaults" do - @db_opts[:local_file] = local_backup - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - expect(EvmDatabaseOps.backup(@db_opts, {})).to eq(local_backup) - end - - it "splits files with a local file" do - @db_opts[:local_file] = local_backup - @db_opts[:byte_count] = "200M" - - allow(file_storage).to receive(:send).with(:add, local_backup, "200M").and_yield(input_path) - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - EvmDatabaseOps.backup(@db_opts, {}) - end - - it "without enough free space" do - EvmSpecHelper.create_guid_miq_server_zone - allow(file_storage).to receive(:class).and_return(MiqSmbSession) - allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes) - allow(EvmDatabaseOps).to receive(:database_size).and_return(200.megabytes) - expect { EvmDatabaseOps.backup(@db_opts, @connect_opts) }.to raise_error(MiqException::MiqDatabaseBackupInsufficientSpace) - expect(MiqQueue.where(:class_name => "MiqEvent", :method_name => "raise_evm_event").count).to eq(1) - end - - it "remotely" do - @db_opts[:local_file] = nil - @connect_opts[:remote_file_name] = "custom_backup" - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to eq("smb://myserver.com/share/db_backup/custom_backup") - end - - it "remotely without a remote file name" do - @db_opts[:local_file] = nil - @connect_opts[:remote_file_name] = nil - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to match(/smb:\/\/myserver.com\/share\/db_backup\/miq_backup_.*/) - end - - it "properly logs the result with no :dbname provided" do - @db_opts.delete(:dbname) - @db_opts[:local_file] = nil - @connect_opts[:remote_file_name] = nil - run_db_ops[:dbname] = "vmdb_production" - allow(described_class).to receive(:backup_file_name).and_return("miq_backup") - expect(PostgresAdmin).to receive(:backup).with(run_db_ops) - - allow(file_storage).to receive(:class).and_return(MiqSmbSession) - - log_stub = instance_double("_log") - expect(described_class).to receive(:_log).twice.and_return(log_stub) - expect(log_stub).to receive(:info).with(any_args) - expect(log_stub).to receive(:info).with("[vmdb_production] database has been backed up to file: [smb://myserver.com/share/db_backup/miq_backup]") - - EvmDatabaseOps.backup(@db_opts, @connect_opts) - end - end - - context "#dump" do - let(:local_dump) { "/tmp/dump_1" } - before do - @connect_opts = {:username => 'blah', :password => 'blahblah', :uri => "smb://myserver.com/share"} - @db_opts = {:dbname => 'vmdb_production', :username => 'root'} - allow(file_storage).to receive(:settings_mount_point).and_return(tmpdir) - allow(file_storage).to receive(:uri_to_local_path).and_return(tmpdir.join("share").to_s) - allow(file_storage).to receive(:add).and_yield(input_path) - - allow(PostgresAdmin).to receive(:run_command_with_logging) - allow(FileUtils).to receive(:mv).and_return(true) - allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(200.megabytes) - allow(EvmDatabaseOps).to receive(:database_size).and_return(100.megabytes) - - MiqEvent # ensure the MiqEvent class is defined so we can verify raised events - end - - it "locally" do - @db_opts[:local_file] = local_dump - expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) - expect(EvmDatabaseOps.dump(@db_opts, @connect_opts)).to eq(local_dump) - end - - it "defaults" do - @db_opts[:local_file] = local_dump - expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) - expect(EvmDatabaseOps.dump(@db_opts, {})).to eq(local_dump) - end - - it "splits files with a local file" do - @db_opts[:local_file] = local_dump - @db_opts[:byte_count] = "200M" - - allow(file_storage).to receive(:send).with(:add, local_dump, "200M").and_yield(input_path) - expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) - EvmDatabaseOps.dump(@db_opts, {}) - end - - it "without enough free space" do - EvmSpecHelper.create_guid_miq_server_zone - allow(file_storage).to receive(:class).and_return(MiqSmbSession) - allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes) - allow(EvmDatabaseOps).to receive(:database_size).and_return(200.megabytes) - expect(PostgresAdmin).to receive(:backup_pg_dump).never - expect { EvmDatabaseOps.dump(@db_opts, @connect_opts) }.to raise_error(MiqException::MiqDatabaseBackupInsufficientSpace) - expect(MiqQueue.where(:class_name => "MiqEvent", :method_name => "raise_evm_event").count).to eq(1) - end - - it "remotely" do - @db_opts[:local_file] = nil - @connect_opts[:remote_file_name] = "custom_pg_dump" - expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) - expect(EvmDatabaseOps.dump(@db_opts, @connect_opts)).to eq("smb://myserver.com/share/db_dump/custom_pg_dump") - end - - it "remotely without a remote file name" do - @db_opts[:local_file] = nil - @connect_opts[:remote_file_name] = nil - expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) - expect(EvmDatabaseOps.dump(@db_opts, @connect_opts)).to match(/smb:\/\/myserver.com\/share\/db_dump\/miq_pg_dump_.*/) - end - end - - context "#restore" do - before do - @connect_opts = {:username => 'blah', :password => 'blahblah'} - @db_opts = {:dbname => 'vmdb_production', :username => 'root'} - - allow(file_storage).to receive(:settings_mount_point).and_return(tmpdir) - allow(file_storage).to receive(:magic_number_for).and_return(:pgdump) - allow(file_storage).to receive(:download).and_yield(input_path) - - allow(PostgresAdmin).to receive(:run_command_with_logging) - - allow(VmdbDatabaseConnection).to receive(:count).and_return(1) - end - - it "from local backup" do - @db_opts[:local_file] = local_backup - expect(EvmDatabaseOps.restore(@db_opts, @connect_opts)).to eq(local_backup) - end - - it "from smb backup" do - @db_opts[:local_file] = nil - remote_backup = "smb://myserver.com/share/pg_backup1.backup" - @connect_opts[:uri] = remote_backup - expect(EvmDatabaseOps.restore(@db_opts, @connect_opts)).to eq(remote_backup) - end - - it "properly logs the result with no :dbname provided" do - @db_opts.delete(:dbname) - @db_opts[:local_file] = nil - remote_backup = "smb://myserver.com/share/pg_backup1.backup" - @connect_opts[:uri] = remote_backup - - log_stub = instance_double("_log") - expect(described_class).to receive(:_log).and_return(log_stub) - expect(log_stub).to receive(:info).with("[vmdb_production] database has been restored from file: [smb://myserver.com/share/pg_backup1.backup]") - - EvmDatabaseOps.restore(@db_opts, @connect_opts) - end - end - - describe "with_file_storage (private method)" do - let(:db_opts) { {} } - let(:connect_opts) { {} } - - # convenience_wrapper for private method - def execute_with_file_storage(action = :backup) - described_class.send(:with_file_storage, action, db_opts, connect_opts) do |dbopts| - yield dbopts if block_given? - end - end - - shared_examples "default with_file_storage behaviors" do - it "sets dbopts[:local_file] to the input_path" do - execute_with_file_storage do |dbopts| - expect(dbopts[:local_file]).to eq(input_path) - end - end - - it "updates db_opts for the block to set the :dbname" do - execute_with_file_storage do |dbopts| - expect(dbopts[:dbname]).to eq("vmdb_production") - end - end - - context "db_opts[:dbname] is set" do - it "does not update :dbname if passed" do - db_opts[:dbname] = "my_db" - - execute_with_file_storage do |dbopts| - expect(dbopts[:dbname]).to eq("my_db") - end - end - end - end - - context "with a local file" do - let(:db_opts) { { :local_file => "/tmp/foo" } } - - before { expect(file_storage).to receive(:add).and_yield(input_path) } - - include_examples "default with_file_storage behaviors" - - it "always uses a file_storage interface" do - execute_with_file_storage do - expect(file_storage).to receive(:test_method) - file_storage.test_method - end - end - - it "does not try to close a session" do - expect(file_storage).to_not receive(:disconnect) - - execute_with_file_storage - end - - it "updates the db_opts[:local_file] to the file_storage fifo" do - execute_with_file_storage do |dbopts| - expect(dbopts[:local_file]).to eq(input_path) - end - end - - it "returns the result of the block" do - expect(execute_with_file_storage { "block result" }).to eq(db_opts[:local_file]) - end - end - - context "without a local file" do - let(:connect_opts) { { :uri => "smb://tmp/foo" } } - - before { allow(file_storage).to receive(:add).and_yield(input_path) } - - include_examples "default with_file_storage behaviors" - - context "for a backup-ish action" do - let(:backup_file) { "/tmp/bar/baz" } - - before { allow(described_class).to receive(:backup_file_name).and_return(backup_file) } - - it "updates db_opts[:local_file] in the method context" do - expected_uri = "smb://tmp/foo/db_backup/baz" - - expect(file_storage).to receive(:send).with(:add, expected_uri) - execute_with_file_storage - end - - it "respects user passed in connect_opts[:remote_file_name]" do - expected_uri = "smb://tmp/foo/db_backup/my_dir/my_backup" - connect_opts[:remote_file_name] = "/my_dir/my_backup" - - expect(file_storage).to receive(:send).with(:add, expected_uri) - execute_with_file_storage - end - - it "returns calculated uri" do - expect(execute_with_file_storage { "block result" }).to eq("smb://tmp/foo/db_backup/baz") - end - - it "yields `db_opt`s only" do - allow(file_storage).to receive(:download) { |&block| block.call(input_path) } - expect do |rspec_probe| - described_class.send(:with_file_storage, :backup, db_opts, connect_opts, &rspec_probe) - end.to yield_with_args(:dbname => "vmdb_production", :local_file => input_path) - end - end - - context "for a restore action" do - before { expect(file_storage).to receive(:magic_number_for).and_return(:pgdump) } - - it "updates db_opts[:local_file] in the method context" do - expect(file_storage).to receive(:send).with(:download, nil, "smb://tmp/foo") - execute_with_file_storage(:restore) - end - - it "parses the dirname of the `uri` and passes that in `connect_opts`" do - expected_connect_opts = { :uri => "smb://tmp/" } - allow(file_storage).to receive(:download) - expect(MiqFileStorage).to receive(:with_interface_class).with(expected_connect_opts) - execute_with_file_storage(:restore) - end - - it "returns calculated uri" do - allow(file_storage).to receive(:download).and_yield(input_path) - expect(execute_with_file_storage(:restore) { "block result" }).to eq("smb://tmp/foo") - end - - it "yields `backup_type` along with `db_opt`s" do - allow(file_storage).to receive(:download) { |&block| block.call(input_path) } - expected_yield_args = [ - { :dbname => "vmdb_production", :local_file => input_path }, - :pgdump - ] - expect do |rspec_probe| - described_class.send(:with_file_storage, :restore, db_opts, connect_opts, &rspec_probe) - end.to yield_with_args(*expected_yield_args) - end - - context "with query_params in the URI" do - let(:connect_opts) { { :uri => "swift://container/foo.gz?2plus2=5" } } - - it "retains query_params when parsing dirname" do - expected_connect_opts = { :uri => "swift://container/?2plus2=5" } - allow(file_storage).to receive(:download) - expect(MiqFileStorage).to receive(:with_interface_class).with(expected_connect_opts) - execute_with_file_storage(:restore) - end - end - end - end - end -end