From 2f258f2578711968bdda063f8dd902782deea2dc Mon Sep 17 00:00:00 2001 From: Niklas Kunz Date: Sat, 25 Jun 2022 19:19:04 +0200 Subject: [PATCH] Format all Python code with `black` --- __main__.py | 4 +- bmaptools/BmapCopy.py | 221 +++++++++++++++---------- bmaptools/BmapCreate.py | 71 +++++---- bmaptools/BmapHelpers.py | 33 ++-- bmaptools/CLI.py | 211 ++++++++++++++---------- bmaptools/Filemap.py | 159 +++++++++++------- bmaptools/TransRead.py | 212 ++++++++++++++---------- contrib/bmap_write.py | 14 +- setup.py | 28 ++-- tests/helpers.py | 94 ++++++----- tests/oldcodebase/BmapCopy1_0.py | 266 +++++++++++++++++-------------- tests/oldcodebase/BmapCopy2_0.py | 244 ++++++++++++++++------------ tests/oldcodebase/BmapCopy2_1.py | 244 ++++++++++++++++------------ tests/oldcodebase/BmapCopy2_2.py | 244 ++++++++++++++++------------ tests/oldcodebase/BmapCopy2_3.py | 264 +++++++++++++++++------------- tests/oldcodebase/BmapCopy2_4.py | 264 +++++++++++++++++------------- tests/oldcodebase/BmapCopy2_5.py | 190 +++++++++++++--------- tests/oldcodebase/BmapCopy2_6.py | 190 +++++++++++++--------- tests/oldcodebase/BmapCopy3_0.py | 201 ++++++++++++++--------- tests/test_CLI.py | 76 ++++++--- tests/test_api_base.py | 131 ++++++++------- tests/test_bmap_helpers.py | 58 ++++--- tests/test_compat.py | 28 ++-- tests/test_filemap.py | 57 ++++--- 24 files changed, 2066 insertions(+), 1438 deletions(-) diff --git a/__main__.py b/__main__.py index ea24b7b..19dd08a 100755 --- a/__main__.py +++ b/__main__.py @@ -6,6 +6,6 @@ from bmaptools.CLI import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe|\.pyz)?$', '', sys.argv[0]) +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe|\.pyz)?$", "", sys.argv[0]) sys.exit(main()) diff --git a/bmaptools/BmapCopy.py b/bmaptools/BmapCopy.py index ca22aba..350e7dc 100644 --- a/bmaptools/BmapCopy.py +++ b/bmaptools/BmapCopy.py @@ -80,6 +80,7 @@ class Error(Exception): support only one type of exceptions, and we basically throw human-readable problem description in case of errors. """ + pass @@ -189,9 +190,11 @@ def __init__(self, image, dest, bmap=None, image_size=None): self._bmap_cs_attrib_name = None # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -231,8 +234,9 @@ def set_psplash_pipe(self, path): if os.path.exists(path) and stat.S_ISFIFO(os.stat(path).st_mode): self._psplash_pipe = path else: - _log.warning("'%s' is not a pipe, so psplash progress will not be " - "updated" % path) + _log.warning( + "'%s' is not a pipe, so psplash progress will not be " "updated" % path + ) def set_progress_indicator(self, file_obj, format_string): """ @@ -259,9 +263,11 @@ def _set_image_size(self, image_size): """ if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -284,13 +290,12 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the checksum # value stored in the file with all zeroes. For these purposes we # create private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access=mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) chksum_pos = mapped_bmap.find(correct_chksum.encode()) assert chksum_pos != -1 - mapped_bmap[chksum_pos:chksum_pos + self._cs_len] = b'0' * self._cs_len + mapped_bmap[chksum_pos : chksum_pos + self._cs_len] = b"0" * self._cs_len hash_obj = hashlib.new(self._cs_type) hash_obj.update(mapped_bmap) @@ -299,9 +304,11 @@ def _verify_bmap_checksum(self): mapped_bmap.close() if calculated_chksum != correct_chksum: - raise Error("checksum mismatch for bmap file '%s': calculated " - "'%s', should be '%s'" - % (self._bmap_path, calculated_chksum, correct_chksum)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_chksum, correct_chksum) + ) def _parse_bmap(self): """ @@ -318,20 +325,24 @@ def _parse_bmap(self): if num >= err.position[0] - 4 and num <= err.position[0] + 4: xml_extract += "Line %d: %s" % (num, line) - raise Error("cannot parse the bmap file '%s' which should be a " - "proper XML file: %s, the XML extract:\n%s" % - (self._bmap_path, err, xml_extract)) + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s, the XML extract:\n%s" + % (self._bmap_path, err, xml_extract) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) - if self.bmap_version_major > int(SUPPORTED_BMAP_VERSION.split('.', 1)[0]): - raise Error("only bmap format version up to %d is supported, " - "version %d is not supported" - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) + if self.bmap_version_major > int(SUPPORTED_BMAP_VERSION.split(".", 1)[0]): + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -345,12 +356,15 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " - "blocks count (%d bytes != %d blocks * %d bytes)" - % (self.image_size, self.blocks_cnt, self.block_size)) - - if self.bmap_version_major > 1 or \ - (self.bmap_version_major == 1 and self.bmap_version_minor == 4): + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) + + if self.bmap_version_major > 1 or ( + self.bmap_version_major == 1 and self.bmap_version_minor == 4 + ): # In bmap format version 1.0-1.3 the only supported checksum type # was SHA1. Version 2.0 started supporting arbitrary checksum # types. A new "ChecksumType" tag was introduce to specify the @@ -377,8 +391,9 @@ def _parse_bmap(self): try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: - raise Error("cannot initialize hash function \"%s\": %s" % - (self._cs_type, err)) + raise Error( + 'cannot initialize hash function "%s": %s' % (self._cs_type, err) + ) self._verify_bmap_checksum() def _update_progress(self, blocks_written): @@ -391,14 +406,16 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - _log.debug("wrote %d blocks out of %d (%d%%)" % - (blocks_written, self.mapped_cnt, percent)) + _log.debug( + "wrote %d blocks out of %d (%d%%)" + % (blocks_written, self.mapped_cnt, percent) + ) else: _log.debug("wrote %d blocks" % blocks_written) if self._progress_file: if self.mapped_cnt: - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() @@ -407,8 +424,8 @@ def _update_progress(self, blocks_written): return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -418,7 +435,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -431,7 +448,7 @@ def _update_progress(self, blocks_written): if self._psplash_pipe and self.mapped_cnt: try: mode = os.O_WRONLY | os.O_NONBLOCK - with os.fdopen(os.open(self._psplash_pipe, mode), 'w') as p_fo: + with os.fdopen(os.open(self._psplash_pipe, mode), "w") as p_fo: p_fo.write("PROGRESS %d\n" % percent) except: pass @@ -473,7 +490,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -536,13 +553,15 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " - "image file '%s': %s" - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: - _log.debug("no more data to read from file '%s'", - self._image_path) + _log.debug( + "no more data to read from file '%s'", self._image_path + ) self._batch_queue.put(None) return @@ -550,17 +569,19 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) // self.block_size - _log.debug("queueing %d blocks, queue length is %d" % - (blocks, self._batch_queue.qsize())) + _log.debug( + "queueing %d blocks, queue length is %d" + % (blocks, self._batch_queue.qsize()) + ) - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and chksum and hash_obj.hexdigest() != chksum: - raise Error("checksum mismatch for blocks range %d-%d: " - "calculated %s, should be %s (image file %s)" - % (first, last, hash_obj.hexdigest(), - chksum, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), chksum, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -582,7 +603,7 @@ def copy(self, sync=True, verify=True): # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -598,8 +619,7 @@ def copy(self, sync=True, verify=True): try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) # Read the image in '_batch_blocks' chunks and write them to the # destination file @@ -630,11 +650,13 @@ def copy(self, sync=True, verify=True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -646,19 +668,25 @@ def copy(self, sync=True, verify=True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " - "have %u - bmap file '%s' does not belong to this " - "image" - % (blocks_written, self._image_path, self._dest_path, - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this " + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -678,8 +706,9 @@ def sync(self): try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): @@ -713,19 +742,29 @@ def __init__(self, image, dest, bmap=None, image_size=None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " - "fit the block device '%s' which has %s capacity" - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" % \ - (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we @@ -754,16 +793,18 @@ def _tune_block_device(self): f_scheduler.seek(0) f_scheduler.write("noop") except IOError as err: - _log.debug("failed to enable I/O optimization, expect " - "suboptimal speed (reason: cannot switch to the " - "'noop' I/O scheduler: %s or blk-mq in use)" % err) + _log.debug( + "failed to enable I/O optimization, expect " + "suboptimal speed (reason: cannot switch to the " + "'noop' I/O scheduler: %s or blk-mq in use)" % err + ) else: # The file contains a list of schedulers with the current # scheduler in square brackets, e.g., "noop deadline [cfq]". # Fetch the name of the current scheduler. import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -776,9 +817,11 @@ def _tune_block_device(self): f_ratio.seek(0) f_ratio.write("1") except IOError as err: - _log.warning("failed to disable excessive buffering, expect " - "worse system responsiveness (reason: cannot set " - "max. I/O ratio to 1: %s)" % err) + _log.warning( + "failed to disable excessive buffering, expect " + "worse system responsiveness (reason: cannot set " + "max. I/O ratio to 1: %s)" % err + ) def _restore_bdev_settings(self): """ @@ -791,16 +834,20 @@ def _restore_bdev_settings(self): with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) def copy(self, sync=True, verify=True): """ diff --git a/bmaptools/BmapCreate.py b/bmaptools/BmapCreate.py index f4fb413..c25b67d 100644 --- a/bmaptools/BmapCreate.py +++ b/bmaptools/BmapCreate.py @@ -65,8 +65,7 @@ # 3.0, and was only fixed in bmap-tools v3.1. SUPPORTED_BMAP_VERSION = "2.0" -_BMAP_START_TEMPLATE = \ - """ +_BMAP_START_TEMPLATE = """ \n" % (' ' * len(self.image_size_human), - ' ' * len("100.0%")) + xml = "%s or %s -->\n" % ( + " " * len(self.image_size_human), + " " * len("100.0%"), + ) xml += " " self._f_bmap.write(xml) self._mapped_count_pos2 = self._f_bmap.tell() - xml = "%s \n\n" % (' ' * len(str(self.blocks_cnt))) + xml = "%s \n\n" % (" " * len(str(self.blocks_cnt))) # pylint: disable=C0301 xml += " \n" xml += " %s \n\n" % self._cs_type xml += " \n" + xml += ' the checksum has be zero (all ASCII "0" symbols). -->\n' xml += " " self._f_bmap.write(xml) self._chksum_pos = self._f_bmap.tell() xml = "0" * self._cs_len + " \n\n" - xml += " \n" xml += " \n" @@ -274,8 +283,9 @@ def _bmap_file_end(self): self._f_bmap.write(xml) self._f_bmap.seek(self._mapped_count_pos1) - self._f_bmap.write("%s or %.1f%%" - % (self.mapped_size_human, self.mapped_percent)) + self._f_bmap.write( + "%s or %.1f%%" % (self.mapped_size_human, self.mapped_percent) + ) self._f_bmap.seek(self._mapped_count_pos2) self._f_bmap.write("%u" % self.mapped_cnt) @@ -330,16 +340,16 @@ def generate(self, include_checksums=True): self.mapped_cnt += last - first + 1 if include_checksums: chksum = self._calculate_chksum(first, last) - chksum = " chksum=\"%s\"" % chksum + chksum = ' chksum="%s"' % chksum else: chksum = "" if first != last: - self._f_bmap.write(" %s-%s \n" - % (chksum, first, last)) + self._f_bmap.write( + " %s-%s \n" % (chksum, first, last) + ) else: - self._f_bmap.write(" %s \n" - % (chksum, first)) + self._f_bmap.write(" %s \n" % (chksum, first)) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) @@ -350,7 +360,6 @@ def generate(self, include_checksums=True): try: self._f_bmap.flush() except IOError as err: - raise Error("cannot flush the bmap file '%s': %s" - % (self._bmap_path, err)) + raise Error("cannot flush the bmap file '%s': %s" % (self._bmap_path, err)) self._f_image.seek(image_pos) diff --git a/bmaptools/BmapHelpers.py b/bmaptools/BmapHelpers.py index 878ed08..ae1d017 100644 --- a/bmaptools/BmapHelpers.py +++ b/bmaptools/BmapHelpers.py @@ -25,12 +25,15 @@ from subprocess import PIPE # Path to check for zfs compatibility. -ZFS_COMPAT_PARAM_PATH = '/sys/module/zfs/parameters/zfs_dmu_offset_next_sync' +ZFS_COMPAT_PARAM_PATH = "/sys/module/zfs/parameters/zfs_dmu_offset_next_sync" + class Error(Exception): """A class for all the other exceptions raised by this module.""" + pass + def human_size(size): """Transform size in bytes into a human-readable form.""" if size == 1: @@ -44,7 +47,8 @@ def human_size(size): if size < 1024: return "%.1f %s" % (size, modifier) - return "%.1f %s" % (size, 'EiB') + return "%.1f %s" % (size, "EiB") + def human_time(seconds): """Transform time in seconds to the HH:MM:SS format.""" @@ -59,6 +63,7 @@ def human_time(seconds): return result + "%.1fs" % seconds + def get_block_size(file_obj): """ Return block size for file object 'file_obj'. Errors are indicated by the @@ -68,18 +73,19 @@ def get_block_size(file_obj): # Get the block size of the host file-system for the image file by calling # the FIGETBSZ ioctl (number 2). try: - binary_data = ioctl(file_obj, 2, struct.pack('I', 0)) - bsize = struct.unpack('I', binary_data)[0] + binary_data = ioctl(file_obj, 2, struct.pack("I", 0)) + bsize = struct.unpack("I", binary_data)[0] if not bsize: raise IOError("get 0 bsize by FIGETBSZ ioctl") except IOError as err: stat = os.fstat(file_obj.fileno()) - if hasattr(stat, 'st_blksize'): + if hasattr(stat, "st_blksize"): bsize = stat.st_blksize else: raise IOError("Unable to determine block size") return bsize + def program_is_available(name): """ This is a helper function which check if the external program 'name' is @@ -93,6 +99,7 @@ def program_is_available(name): return False + def get_file_system_type(path): """Return the file system type for 'path'.""" @@ -112,11 +119,14 @@ def get_file_system_type(path): ftype = fields[1].lower() if not ftype: - raise Error("failed to find file system type for path at '%s'\n" - "Here is the 'df -T' output\nstdout:\n%s\nstderr:\n%s" - % (path, stdout, stderr)) + raise Error( + "failed to find file system type for path at '%s'\n" + "Here is the 'df -T' output\nstdout:\n%s\nstderr:\n%s" + % (path, stdout, stderr) + ) return ftype + def is_zfs_configuration_compatible(): """Return if hosts zfs configuration is compatible.""" @@ -128,11 +138,10 @@ def is_zfs_configuration_compatible(): with open(path, "r") as fobj: return int(fobj.readline()) == 1 except IOError as err: - raise Error("cannot open zfs param path '%s': %s" - % (path, err)) + raise Error("cannot open zfs param path '%s': %s" % (path, err)) except ValueError as err: - raise Error("invalid value read from param path '%s': %s" - % (path, err)) + raise Error("invalid value read from param path '%s': %s" % (path, err)) + def is_compatible_file_system(path): """Return if paths file system is compatible.""" diff --git a/bmaptools/CLI.py b/bmaptools/CLI.py index cc5ab2f..df351be 100644 --- a/bmaptools/CLI.py +++ b/bmaptools/CLI.py @@ -45,6 +45,7 @@ log = logging.getLogger() # pylint: disable=C0103 + def print_error_with_tb(msgformat, *args): """Print an error message occurred along with the traceback.""" @@ -98,6 +99,7 @@ def __init__(self, file_obj, name): def __getattr__(self, name): return getattr(self._file_obj, name) + def open_block_device(path): """ This is a helper function for 'open_files()' which is called if the @@ -115,8 +117,7 @@ def open_block_device(path): try: descriptor = os.open(path, os.O_WRONLY | os.O_EXCL) except OSError as err: - error_out("cannot open block device '%s' in exclusive mode: %s", - path, err) + error_out("cannot open block device '%s' in exclusive mode: %s", path, err) # Turn the block device file descriptor into a file object try: @@ -127,6 +128,7 @@ def open_block_device(path): return NamedFile(file_obj, path) + def report_verification_results(context, sigs): """ This is a helper function which reports the GPG signature verification @@ -140,13 +142,19 @@ def report_verification_results(context, sigs): if (sig.summary & gpg.constants.SIGSUM_VALID) != 0: key = context.get_key(sig.fpr) author = "%s <%s>" % (key.uids[0].name, key.uids[0].email) - log.info("successfully verified bmap file signature of %s " - "(fingerprint %s)" % (author, sig.fpr)) + log.info( + "successfully verified bmap file signature of %s " + "(fingerprint %s)" % (author, sig.fpr) + ) else: - error_out("signature verification failed (fingerprint %s): %s\n" - "Either fix the problem or use --no-sig-verify to " - "disable signature verification", - sig.fpr, sig.status[2].lower()) + error_out( + "signature verification failed (fingerprint %s): %s\n" + "Either fix the problem or use --no-sig-verify to " + "disable signature verification", + sig.fpr, + sig.status[2].lower(), + ) + def verify_detached_bmap_signature(args, bmap_obj, bmap_path): """ @@ -161,8 +169,7 @@ def verify_detached_bmap_signature(args, bmap_obj, bmap_path): try: sig_obj = TransRead.TransRead(args.bmap_sig) except TransRead.Error as err: - error_out("cannot open bmap signature file '%s':\n%s", - args.bmap_sig, err) + error_out("cannot open bmap signature file '%s':\n%s", args.bmap_sig, err) sig_path = args.bmap_sig else: # Check if there is a stand-alone signature file @@ -184,8 +191,7 @@ def verify_detached_bmap_signature(args, bmap_obj, bmap_path): try: tmp_obj = tempfile.NamedTemporaryFile("wb+") except IOError as err: - error_out("cannot create a temporary file for the signature:\n%s", - err) + error_out("cannot create a temporary file for the signature:\n%s", err) shutil.copyfileobj(sig_obj, tmp_obj) tmp_obj.seek(0) @@ -195,9 +201,11 @@ def verify_detached_bmap_signature(args, bmap_obj, bmap_path): try: import gpg except ImportError: - error_out("cannot verify the signature because the python \"gpg\" " - "module is not installed on your system\nPlease, either " - "install the module or use --no-sig-verify") + error_out( + 'cannot verify the signature because the python "gpg" ' + "module is not installed on your system\nPlease, either " + "install the module or use --no-sig-verify" + ) try: context = gpg.Context() @@ -205,17 +213,22 @@ def verify_detached_bmap_signature(args, bmap_obj, bmap_path): signed_data = io.FileIO(bmap_obj.name) sigs = context.verify(signed_data, signature, None)[1].signatures except gpg.errors.GPGMEError as err: - error_out("failure when trying to verify GPG signature: %s\n" - "Make sure file \"%s\" has proper GPG format", - err.getstring(), sig_path) + error_out( + "failure when trying to verify GPG signature: %s\n" + 'Make sure file "%s" has proper GPG format', + err.getstring(), + sig_path, + ) except gpg.errors.BadSignatures as err: error_out("discovered a BAD GPG signature: %s\n", sig_path) sig_obj.close() if len(sigs) == 0: - log.warning("the \"%s\" signature file does not actually contain " - "any valid signatures" % sig_path) + log.warning( + 'the "%s" signature file does not actually contain ' + "any valid signatures" % sig_path + ) else: report_verification_results(context, sigs) @@ -229,16 +242,20 @@ def verify_clearsign_bmap_signature(args, bmap_obj): """ if args.bmap_sig: - error_out("the bmap file has clearsign format and already contains " - "the signature, so --bmap-sig option should not be used") + error_out( + "the bmap file has clearsign format and already contains " + "the signature, so --bmap-sig option should not be used" + ) try: import gpg except ImportError: - error_out("cannot verify the signature because the python \"gpg\"" - "module is not installed on your system\nCannot extract " - "block map from the bmap file which has clearsign format, " - "please, install the module") + error_out( + 'cannot verify the signature because the python "gpg"' + "module is not installed on your system\nCannot extract " + "block map from the bmap file which has clearsign format, " + "please, install the module" + ) try: context = gpg.Context() @@ -246,16 +263,20 @@ def verify_clearsign_bmap_signature(args, bmap_obj): plaintext = io.BytesIO() sigs = context.verify(plaintext, signature, None) except gpg.errors.GPGMEError as err: - error_out("failure when trying to verify GPG signature: %s\n" - "make sure the bmap file has proper GPG format", - err[2].lower()) + error_out( + "failure when trying to verify GPG signature: %s\n" + "make sure the bmap file has proper GPG format", + err[2].lower(), + ) except gpg.errors.BadSignatures as err: error_out("discovered a BAD GPG signature: %s\n", sig_path) if not args.no_sig_verify: if len(sigs) == 0: - log.warning("the bmap file clearsign signature does not actually " - "contain any valid signatures") + log.warning( + "the bmap file clearsign signature does not actually " + "contain any valid signatures" + ) else: report_verification_results(context, sigs) @@ -346,7 +367,7 @@ def find_and_open_bmap(args): pass image_path, ext = os.path.splitext(image_path) - if ext == '': + if ext == "": return (None, None) if not bmap_obj.is_url: @@ -393,8 +414,10 @@ def open_files(args): # Most probably the user specified the bmap file instead of the image # file by mistake. bmap_obj.close() - error_out("Make sure you are writing your image and not the bmap file " - "(you specified the same path for them)") + error_out( + "Make sure you are writing your image and not the bmap file " + "(you specified the same path for them)" + ) # If the destination file is under "/dev", but does not exist, print a # warning. This is done in order to be more user-friendly, because @@ -405,17 +428,21 @@ def open_files(args): # the destination file is not a special device for some reasons. if os.path.normpath(args.dest).startswith("/dev/"): if not os.path.exists(args.dest): - log.warning("\"%s\" does not exist, creating a regular file " - "\"%s\"" % (args.dest, args.dest)) + log.warning( + '"%s" does not exist, creating a regular file ' + '"%s"' % (args.dest, args.dest) + ) elif stat.S_ISREG(os.stat(args.dest).st_mode): - log.warning("\"%s\" is under \"/dev\", but it is a regular file, " - "not a device node" % args.dest) + log.warning( + '"%s" is under "/dev", but it is a regular file, ' + "not a device node" % args.dest + ) # Try to open the destination file. If it does not exist, a new regular # file will be created. If it exists and it is a regular file - it'll be # truncated. If this is a block device, it'll just be opened. try: - dest_obj = open(args.dest, 'wb+') + dest_obj = open(args.dest, "wb+") except IOError as err: error_out("cannot open destination file '%s':\n%s", args.dest, err) @@ -425,8 +452,7 @@ def open_files(args): dest_obj.close() dest_obj = open_block_device(args.dest) - return (image_obj, dest_obj, bmap_obj, bmap_path, image_obj.size, - dest_is_blkdev) + return (image_obj, dest_obj, bmap_obj, bmap_path, image_obj.size, dest_is_blkdev) def copy_command(args): @@ -438,12 +464,14 @@ def copy_command(args): if args.bmap_sig and args.no_sig_verify: error_out("--bmap-sig and --no-sig-verify cannot be used together") - image_obj, dest_obj, bmap_obj, bmap_path, image_size, dest_is_blkdev = \ - open_files(args) + image_obj, dest_obj, bmap_obj, bmap_path, image_size, dest_is_blkdev = open_files( + args + ) if args.bmap_sig and not bmap_obj: - error_out("the bmap signature file was specified, but bmap file was " - "not found") + error_out( + "the bmap signature file was specified, but bmap file was " "not found" + ) f_obj = verify_bmap_signature(args, bmap_obj, bmap_path) if f_obj: @@ -457,18 +485,20 @@ def copy_command(args): if dest_is_blkdev: dest_str = "block device '%s'" % args.dest # For block devices, use the specialized class - writer = BmapCopy.BmapBdevCopy(image_obj, dest_obj, bmap_obj, - image_size) + writer = BmapCopy.BmapBdevCopy(image_obj, dest_obj, bmap_obj, image_size) else: dest_str = "file '%s'" % os.path.basename(args.dest) - writer = BmapCopy.BmapCopy(image_obj, dest_obj, bmap_obj, - image_size) + writer = BmapCopy.BmapCopy(image_obj, dest_obj, bmap_obj, image_size) except BmapCopy.Error as err: error_out(err) # Print the progress indicator while copying - if not args.quiet and not args.debug and \ - sys.stderr.isatty() and sys.stdout.isatty(): + if ( + not args.quiet + and not args.debug + and sys.stderr.isatty() + and sys.stdout.isatty() + ): writer.set_progress_indicator(sys.stderr, "bmaptool: info: %d%% copied") start_time = time.time() @@ -476,17 +506,27 @@ def copy_command(args): if args.nobmap: log.info("no bmap given, copy entire image to '%s'" % args.dest) else: - error_out("bmap file not found, please, use --nobmap option to " - "flash without bmap") + error_out( + "bmap file not found, please, use --nobmap option to " + "flash without bmap" + ) else: log.info("block map format version %s" % writer.bmap_version) - log.info("%d blocks of size %d (%s), mapped %d blocks (%s or %.1f%%)" - % (writer.blocks_cnt, writer.block_size, - writer.image_size_human, writer.mapped_cnt, - writer.mapped_size_human, writer.mapped_percent)) - log.info("copying image '%s' to %s using bmap file '%s'" - % (os.path.basename(args.image), dest_str, - os.path.basename(bmap_path))) + log.info( + "%d blocks of size %d (%s), mapped %d blocks (%s or %.1f%%)" + % ( + writer.blocks_cnt, + writer.block_size, + writer.image_size_human, + writer.mapped_cnt, + writer.mapped_size_human, + writer.mapped_percent, + ) + ) + log.info( + "copying image '%s' to %s using bmap file '%s'" + % (os.path.basename(args.image), dest_str, os.path.basename(bmap_path)) + ) if args.psplash_pipe: writer.set_psplash_pipe(args.psplash_pipe) @@ -508,9 +548,10 @@ def copy_command(args): copying_time = time.time() - start_time copying_speed = writer.mapped_size // copying_time - log.info("copying time: %s, copying speed %s/sec" - % (BmapHelpers.human_time(copying_time), - BmapHelpers.human_size(copying_speed))) + log.info( + "copying time: %s, copying speed %s/sec" + % (BmapHelpers.human_time(copying_time), BmapHelpers.human_size(copying_speed)) + ) dest_obj.close() if bmap_obj: @@ -568,20 +609,22 @@ def create_command(args): sys.stdout.write(output.read()) if creator.mapped_cnt == creator.blocks_cnt: - log.warning("all %s are mapped, no holes in '%s'" - % (creator.image_size_human, args.image)) - log.warning("was the image handled incorrectly and holes " - "were expanded?") + log.warning( + "all %s are mapped, no holes in '%s'" + % (creator.image_size_human, args.image) + ) + log.warning("was the image handled incorrectly and holes " "were expanded?") def parse_arguments(): """A helper function which parses the input arguments.""" text = sys.modules[__name__].__doc__ - parser = argparse.ArgumentParser(description=text, prog='bmaptool') + parser = argparse.ArgumentParser(description=text, prog="bmaptool") # The --version option - parser.add_argument("--version", action="version", - version="%(prog)s " + "%s" % VERSION) + parser.add_argument( + "--version", action="version", version="%(prog)s " + "%s" % VERSION + ) # The --quiet option text = "be quiet" @@ -621,8 +664,9 @@ def parse_arguments(): parser_copy.set_defaults(func=copy_command) # The first positional argument - image file - text = "the image file to copy. Supported formats: uncompressed, " + \ - ", ".join(TransRead.SUPPORTED_COMPRESSION_TYPES) + text = "the image file to copy. Supported formats: uncompressed, " + ", ".join( + TransRead.SUPPORTED_COMPRESSION_TYPES + ) parser_copy.add_argument("image", help=text) # The second positional argument - block device node @@ -663,10 +707,10 @@ def setup_logger(loglevel): """ # Esc-sequences for coloured output - esc_red = '\033[91m' # pylint: disable=W1401 - esc_yellow = '\033[93m' # pylint: disable=W1401 - esc_green = '\033[92m' # pylint: disable=W1401 - esc_end = '\033[0m' # pylint: disable=W1401 + esc_red = "\033[91m" # pylint: disable=W1401 + esc_yellow = "\033[93m" # pylint: disable=W1401 + esc_green = "\033[92m" # pylint: disable=W1401 + esc_end = "\033[0m" # pylint: disable=W1401 class MyFormatter(logging.Formatter): """ @@ -681,8 +725,14 @@ def __init__(self, fmt=None, datefmt=None): self._orig_fmt = self._fmt # Prefix with green-colored time-stamp, as well as with module name # and line number - self._dbg_fmt = "[" + esc_green + "%(asctime)s" + esc_end + \ - "] [%(module)s,%(lineno)d] " + self._fmt + self._dbg_fmt = ( + "[" + + esc_green + + "%(asctime)s" + + esc_end + + "] [%(module)s,%(lineno)d] " + + self._fmt + ) def format(self, record): """ @@ -734,14 +784,15 @@ def main(): traceback.print_exc() log.info("The contents of /proc/meminfo:") - with open('/proc/meminfo', 'rt') as file_obj: + with open("/proc/meminfo", "rt") as file_obj: for line in file_obj: print(line.strip()) log.info("The contents of /proc/self/status:") - with open('/proc/self/status', 'rt') as file_obj: + with open("/proc/self/status", "rt") as file_obj: for line in file_obj: print(line.strip()) + if __name__ == "__main__": sys.exit(main()) diff --git a/bmaptools/Filemap.py b/bmaptools/Filemap.py index 8831b2d..e3fdb60 100644 --- a/bmaptools/Filemap.py +++ b/bmaptools/Filemap.py @@ -44,11 +44,13 @@ class ErrorNotSupp(Exception): An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature is not supported either by the kernel or the file-system. """ + pass class Error(Exception): """A class for all the other exceptions raised by this module.""" + pass @@ -77,37 +79,42 @@ def __init__(self, image): try: self.image_size = os.fstat(self._f_image.fileno()).st_size except IOError as err: - raise Error("cannot get information about file '%s': %s" - % (self._f_image.name, err)) + raise Error( + "cannot get information about file '%s': %s" % (self._f_image.name, err) + ) try: self.block_size = BmapHelpers.get_block_size(self._f_image) except IOError as err: - raise Error("cannot get block size for '%s': %s" - % (self._image_path, err)) + raise Error("cannot get block size for '%s': %s" % (self._image_path, err)) self.blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size try: self._f_image.flush() except IOError as err: - raise Error("cannot flush image file '%s': %s" - % (self._image_path, err)) + raise Error("cannot flush image file '%s': %s" % (self._image_path, err)) try: os.fsync(self._f_image.fileno()), except OSError as err: - raise Error("cannot synchronize image file '%s': %s " - % (self._image_path, err.strerror)) + raise Error( + "cannot synchronize image file '%s': %s " + % (self._image_path, err.strerror) + ) if not BmapHelpers.is_compatible_file_system(self._image_path): fstype = BmapHelpers.get_file_system_type(self._image_path) - raise Error("image file on incompatible file system '%s': '%s': see docs for fix" - % (self._image_path, fstype)) + raise Error( + "image file on incompatible file system '%s': '%s': see docs for fix" + % (self._image_path, fstype) + ) - _log.debug("opened image \"%s\"" % self._image_path) - _log.debug("block size %d, blocks count %d, image size %d" - % (self.block_size, self.blocks_cnt, self.image_size)) + _log.debug('opened image "%s"' % self._image_path) + _log.debug( + "block size %d, blocks count %d, image size %d" + % (self.block_size, self.blocks_cnt, self.image_size) + ) def __del__(self): """The class destructor which just closes the image file.""" @@ -117,10 +124,9 @@ def __del__(self): def _open_image_file(self): """Open the image file.""" try: - self._f_image = open(self._image_path, 'rb') + self._f_image = open(self._image_path, "rb") except IOError as err: - raise Error("cannot open image file '%s': %s" - % (self._image_path, err)) + raise Error("cannot open image file '%s': %s" % (self._image_path, err)) self._f_image_needs_close = True @@ -185,8 +191,10 @@ def _lseek(file_obj, offset, whence): if err.errno == errno.ENXIO: return -1 elif err.errno == errno.EINVAL: - raise ErrorNotSupp("the kernel or file-system does not support " - "\"SEEK_HOLE\" and \"SEEK_DATA\"") + raise ErrorNotSupp( + "the kernel or file-system does not support " + '"SEEK_HOLE" and "SEEK_DATA"' + ) else: raise @@ -226,23 +234,27 @@ def _probe_seek_hole(self): try: tmp_obj = tempfile.TemporaryFile("w+", dir=directory) except OSError as err: - raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" - % (directory, err)) + raise ErrorNotSupp( + 'cannot create a temporary in "%s": %s' % (directory, err) + ) try: os.ftruncate(tmp_obj.fileno(), self.block_size) except OSError as err: - raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s" - % (directory, err)) + raise ErrorNotSupp( + 'cannot truncate temporary file in "%s": %s' % (directory, err) + ) offs = _lseek(tmp_obj, 0, _SEEK_HOLE) if offs != 0: # We are dealing with the stub 'SEEK_HOLE' implementation which # always returns EOF. _log.debug("lseek(0, SEEK_HOLE) returned %d" % offs) - raise ErrorNotSupp("the file-system does not support " - "\"SEEK_HOLE\" and \"SEEK_DATA\" but only " - "provides a stub implementation") + raise ErrorNotSupp( + "the file-system does not support " + '"SEEK_HOLE" and "SEEK_DATA" but only ' + "provides a stub implementation" + ) tmp_obj.close() @@ -252,10 +264,9 @@ def block_is_mapped(self, block): if offs == -1: result = False else: - result = (offs // self.block_size == block) + result = offs // self.block_size == block - _log.debug("FilemapSeek: block_is_mapped(%d) returns %s" - % (block, result)) + _log.debug("FilemapSeek: block_is_mapped(%d) returns %s" % (block, result)) return result def block_is_unmapped(self, block): @@ -286,20 +297,23 @@ def _get_ranges(self, start, count, whence1, whence2): start_blk = start // self.block_size end_blk = end // self.block_size - 1 - _log.debug("FilemapSeek: yielding range (%d, %d)" - % (start_blk, end_blk)) + _log.debug("FilemapSeek: yielding range (%d, %d)" % (start_blk, end_blk)) yield (start_blk, end_blk) def get_mapped_ranges(self, start, count): """Refer the '_FilemapBase' class for the documentation.""" - _log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) + _log.debug( + "FilemapSeek: get_mapped_ranges(%d, %d(%d))" + % (start, count, start + count - 1) + ) return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) def get_unmapped_ranges(self, start, count): """Refer the '_FilemapBase' class for the documentation.""" - _log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) + _log.debug( + "FilemapSeek: get_unmapped_ranges(%d, %d(%d))" + % (start, count, start + count - 1) + ) return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) @@ -356,7 +370,7 @@ def __init__(self, image): self._buf_size += _FIEMAP_SIZE # Allocate a mutable buffer for the FIEMAP ioctl - self._buf = array.array('B', [0] * self._buf_size) + self._buf = array.array("B", [0] * self._buf_size) # Check if the FIEMAP ioctl is supported self.block_is_mapped(0) @@ -372,17 +386,27 @@ def _invoke_fiemap(self, block, count): """ if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt): - raise Error("bad block number %d, should be within [0, %d]" - % (block, self.blocks_cnt)) + raise Error( + "bad block number %d, should be within [0, %d]" + % (block, self.blocks_cnt) + ) # Initialize the 'struct fiemap' part of the buffer. We use the # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is # synchronized. The reason for this is that early FIEMAP # implementations had many bugs related to cached dirty data, and # synchronizing the file is a necessary work-around. - struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size, - count * self.block_size, _FIEMAP_FLAG_SYNC, 0, - self._fiemap_extent_cnt, 0) + struct.pack_into( + _FIEMAP_FORMAT, + self._buf, + 0, + block * self.block_size, + count * self.block_size, + _FIEMAP_FLAG_SYNC, + 0, + self._fiemap_extent_cnt, + 0, + ) try: fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1) @@ -390,17 +414,21 @@ def _invoke_fiemap(self, block, count): # Note, the FIEMAP ioctl is supported by the Linux kernel starting # from version 2.6.28 (year 2008). if err.errno == errno.EOPNOTSUPP: - errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ - "by the file-system" + errstr = ( + "FilemapFiemap: the FIEMAP ioctl is not supported " + "by the file-system" + ) _log.debug(errstr) raise ErrorNotSupp(errstr) if err.errno == errno.ENOTTY: - errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ - "by the kernel" + errstr = ( + "FilemapFiemap: the FIEMAP ioctl is not supported " "by the kernel" + ) _log.debug(errstr) raise ErrorNotSupp(errstr) - raise Error("the FIEMAP ioctl failed for '%s': %s" - % (self._image_path, err)) + raise Error( + "the FIEMAP ioctl failed for '%s': %s" % (self._image_path, err) + ) return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) @@ -412,8 +440,7 @@ def block_is_mapped(self, block): # If it contains zero, the block is not mapped, otherwise it is # mapped. result = bool(struct_fiemap[3]) - _log.debug("FilemapFiemap: block_is_mapped(%d) returns %s" - % (block, result)) + _log.debug("FilemapFiemap: block_is_mapped(%d) returns %s" % (block, result)) return result def block_is_unmapped(self, block): @@ -427,8 +454,9 @@ def _unpack_fiemap_extent(self, index): """ offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index - return struct.unpack(_FIEMAP_EXTENT_FORMAT, - self._buf[offset: offset + _FIEMAP_EXTENT_SIZE]) + return struct.unpack( + _FIEMAP_EXTENT_FORMAT, self._buf[offset : offset + _FIEMAP_EXTENT_SIZE] + ) def _do_get_mapped_ranges(self, start, count): """ @@ -478,8 +506,10 @@ def _do_get_mapped_ranges(self, start, count): def get_mapped_ranges(self, start, count): """Refer the '_FilemapBase' class for the documentation.""" - _log.debug("FilemapFiemap: get_mapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) + _log.debug( + "FilemapFiemap: get_mapped_ranges(%d, %d(%d))" + % (start, count, start + count - 1) + ) iterator = self._do_get_mapped_ranges(start, count) try: @@ -491,31 +521,36 @@ def get_mapped_ranges(self, start, count): if last_prev == first - 1: last_prev = last else: - _log.debug("FilemapFiemap: yielding range (%d, %d)" - % (first_prev, last_prev)) + _log.debug( + "FilemapFiemap: yielding range (%d, %d)" % (first_prev, last_prev) + ) yield (first_prev, last_prev) first_prev, last_prev = first, last - _log.debug("FilemapFiemap: yielding range (%d, %d)" - % (first_prev, last_prev)) + _log.debug("FilemapFiemap: yielding range (%d, %d)" % (first_prev, last_prev)) yield (first_prev, last_prev) def get_unmapped_ranges(self, start, count): """Refer the '_FilemapBase' class for the documentation.""" - _log.debug("FilemapFiemap: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) + _log.debug( + "FilemapFiemap: get_unmapped_ranges(%d, %d(%d))" + % (start, count, start + count - 1) + ) hole_first = start for first, last in self._do_get_mapped_ranges(start, count): if first > hole_first: - _log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, first - 1)) + _log.debug( + "FilemapFiemap: yielding range (%d, %d)" % (hole_first, first - 1) + ) yield (hole_first, first - 1) hole_first = last + 1 if hole_first < start + count: - _log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, start + count - 1)) + _log.debug( + "FilemapFiemap: yielding range (%d, %d)" + % (hole_first, start + count - 1) + ) yield (hole_first, start + count - 1) diff --git a/bmaptools/TransRead.py b/bmaptools/TransRead.py index 68c106e..85ed05b 100644 --- a/bmaptools/TransRead.py +++ b/bmaptools/TransRead.py @@ -52,9 +52,21 @@ # pylint: disable=R0915 # A list of supported compression types -SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'xz', 'lzo', 'lz4', 'zst', - 'tar.gz', 'tar.bz2', 'tar.xz', 'tar.lzo', - 'tar.lz4', 'tar.zst', 'zip') +SUPPORTED_COMPRESSION_TYPES = ( + "bz2", + "gz", + "xz", + "lzo", + "lz4", + "zst", + "tar.gz", + "tar.bz2", + "tar.xz", + "tar.lzo", + "tar.lz4", + "tar.zst", + "zip", +) def _fake_seek_forward(file_obj, cur_pos, offset, whence=os.SEEK_SET): @@ -69,14 +81,16 @@ def _fake_seek_forward(file_obj, cur_pos, offset, whence=os.SEEK_SET): elif whence == os.SEEK_CUR: new_pos = cur_pos + offset else: - raise Error("'seek()' method requires the 'whence' argument " - "to be %d or %d, but %d was passed" - % (os.SEEK_SET, os.SEEK_CUR, whence)) + raise Error( + "'seek()' method requires the 'whence' argument " + "to be %d or %d, but %d was passed" % (os.SEEK_SET, os.SEEK_CUR, whence) + ) if new_pos < cur_pos: - raise Error("''seek()' method supports only seeking forward, " - "seeking from %d to %d is not allowed" - % (cur_pos, new_pos)) + raise Error( + "''seek()' method supports only seeking forward, " + "seeking from %d to %d is not allowed" % (cur_pos, new_pos) + ) length = new_pos - cur_pos to_read = length @@ -88,8 +102,7 @@ def _fake_seek_forward(file_obj, cur_pos, offset, whence=os.SEEK_SET): to_read -= len(buf) if to_read < 0: - raise Error("seeked too far: %d instead of %d" - % (new_pos - to_read, new_pos)) + raise Error("seeked too far: %d instead of %d" % (new_pos - to_read, new_pos)) return new_pos - to_read @@ -100,6 +113,7 @@ class Error(Exception): one type of exceptions, and we basically throw human-readable problem description in case of errors. """ + pass @@ -120,8 +134,10 @@ def _decode_sshpass_exit_code(code): elif code == 5: result = "invalid/incorrect password" elif code == 6: - result = "host public key is unknown. sshpass exits without " \ - "confirming the new key" + result = ( + "host public key is unknown. sshpass exits without " + "confirming the new key" + ) elif code == 255: # SSH result =s 255 on any error result = "ssh error" @@ -148,7 +164,7 @@ def __init__(self, filepath): # unknown self.size = None # Type of the compression of the file - self.compression_type = 'none' + self.compression_type = "none" # Whether the 'bz2file' PyPI module was found self.bz2file_found = False # Whether the file is behind an URL @@ -237,38 +253,39 @@ def _open_compressed_file(self): def is_gzip(name): """Returns 'True' if file 'name' is compressed with 'gzip'.""" - if name.endswith('.gzip') or \ - (name.endswith('.gz') and not name.endswith('.tar.gz')): + if name.endswith(".gzip") or ( + name.endswith(".gz") and not name.endswith(".tar.gz") + ): return True return False def is_bzip2(name): """Returns 'True' if file 'name' is compressed with 'bzip2'.""" - if name.endswith('.bz2') and not name.endswith('.tar.bz2'): + if name.endswith(".bz2") and not name.endswith(".tar.bz2"): return True return False def is_xz(name): """Returns 'True' if file 'name' is compressed with 'xz'.""" - if name.endswith('.xz') and not name.endswith('.tar.xz'): + if name.endswith(".xz") and not name.endswith(".tar.xz"): return True return False def is_lzop(name): """Returns 'True' if file 'name' is compressed with 'lzop'.""" - if name.endswith('.lzo') and not name.endswith('.tar.lzo'): + if name.endswith(".lzo") and not name.endswith(".tar.lzo"): return True return False def is_lz4(name): """Returns 'True' if file 'name' is compressed with 'lz4'.""" - if name.endswith('.lz4') and not name.endswith('.tar.lz4'): + if name.endswith(".lz4") and not name.endswith(".tar.lz4"): return True return False def is_zst(name): """Returns 'True' if file 'name' is compressed with 'zstd'.""" - if name.endswith('.zst') and not name.endswith('.tar.zst'): + if name.endswith(".zst") and not name.endswith(".tar.zst"): return True return False @@ -278,7 +295,7 @@ def is_tar_gz(name): 'gzip'. """ - if name.endswith('.tar.gz') or name.endswith('.tgz'): + if name.endswith(".tar.gz") or name.endswith(".tgz"): return True return False @@ -288,8 +305,12 @@ def is_tar_bz2(name): 'bzip2'. """ - if name.endswith('.tar.bz2') or name.endswith('.tbz') or \ - name.endswith('.tbz2') or name.endswith('.tb2'): + if ( + name.endswith(".tar.bz2") + or name.endswith(".tbz") + or name.endswith(".tbz2") + or name.endswith(".tb2") + ): return True return False @@ -298,7 +319,7 @@ def is_tar_xz(name): Returns 'True' if file 'name' is a tar archive compressed with 'xz'. """ - if name.endswith('.tar.xz') or name.endswith('.txz'): + if name.endswith(".tar.xz") or name.endswith(".txz"): return True return False @@ -308,7 +329,7 @@ def is_tar_lzo(name): 'lzop'. """ - if name.endswith('.tar.lzo') or name.endswith('.tzo'): + if name.endswith(".tar.lzo") or name.endswith(".tzo"): return True return False @@ -318,7 +339,7 @@ def is_tar_lz4(name): 'lz4'. """ - if name.endswith('.tar.lz4') or name.endswith('.tlz4'): + if name.endswith(".tar.lz4") or name.endswith(".tlz4"): return True return False @@ -328,13 +349,13 @@ def is_tar_zst(name): 'zstd'. """ - if name.endswith('.tar.zst') or name.endswith('.tzst'): + if name.endswith(".tar.zst") or name.endswith(".tzst"): return True return False archiver = None if is_tar_gz(self.name) or is_gzip(self.name): - self.compression_type = 'gzip' + self.compression_type = "gzip" if BmapHelpers.program_is_available("pigz"): decompressor = "pigz" else: @@ -346,7 +367,7 @@ def is_tar_zst(name): archiver = "tar" args = "-x -z -O" elif is_tar_bz2(self.name) or is_bzip2(self.name): - self.compression_type = 'bzip2' + self.compression_type = "bzip2" if BmapHelpers.program_is_available("pbzip2"): decompressor = "pbzip2" else: @@ -358,7 +379,7 @@ def is_tar_zst(name): archiver = "tar" args = "-x -j -O" elif is_tar_xz(self.name) or is_xz(self.name): - self.compression_type = 'xz' + self.compression_type = "xz" decompressor = "xz" if is_xz(self.name): args = "-d -c" @@ -366,7 +387,7 @@ def is_tar_zst(name): archiver = "tar" args = "-x -J -O" elif is_tar_lzo(self.name) or is_lzop(self.name): - self.compression_type = 'lzo' + self.compression_type = "lzo" decompressor = "lzop" if is_lzop(self.name): args = "-d -c" @@ -374,11 +395,11 @@ def is_tar_zst(name): archiver = "tar" args = "-x --lzo -O" elif self.name.endswith(".zip"): - self.compression_type = 'zip' + self.compression_type = "zip" decompressor = "funzip" args = "" elif is_tar_lz4(self.name) or is_lz4(self.name): - self.compression_type = 'lz4' + self.compression_type = "lz4" decompressor = "lz4" if is_lz4(self.name): args = "-d -c" @@ -386,7 +407,7 @@ def is_tar_zst(name): archiver = "tar" args = "-x -Ilz4 -O" elif is_tar_zst(self.name) or is_zst(self.name): - self.compression_type = 'zst' + self.compression_type = "zst" decompressor = "zstd" if is_zst(self.name): args = "-d" @@ -405,12 +426,15 @@ def is_tar_zst(name): # Make sure decompressor and the archiver programs are available if not BmapHelpers.program_is_available(decompressor): - raise Error("the \"%s\" program is not available but it is " - "required decompressing \"%s\"" - % (decompressor, self.name)) + raise Error( + 'the "%s" program is not available but it is ' + 'required decompressing "%s"' % (decompressor, self.name) + ) if archiver and not BmapHelpers.program_is_available(archiver): - raise Error("the \"%s\" program is not available but it is " - "required reading \"%s\"" % (archiver, self.name)) + raise Error( + 'the "%s" program is not available but it is ' + 'required reading "%s"' % (archiver, self.name) + ) # Start the decompressor process. We'll send the data to its stdin and # read the decompressed data from its stdout. @@ -424,15 +448,21 @@ def is_tar_zst(name): else: child_stdin = self._f_objs[-1].fileno() - child_process = subprocess.Popen(args, shell=True, - bufsize=1024 * 1024, - stdin=child_stdin, - stdout=subprocess.PIPE) + child_process = subprocess.Popen( + args, + shell=True, + bufsize=1024 * 1024, + stdin=child_stdin, + stdout=subprocess.PIPE, + ) if child_stdin == subprocess.PIPE: # A separate reader thread is created only when we are reading via # urllib2. - args = (self._f_objs[-1], child_process.stdin, ) + args = ( + self._f_objs[-1], + child_process.stdin, + ) self._rthread = threading.Thread(target=self._read_thread, args=args) self._rthread.daemon = True self._rthread.start() @@ -457,32 +487,40 @@ def _open_url_ssh(self, parsed_url): # Make sure the ssh client program is installed if not BmapHelpers.program_is_available("ssh"): - raise Error("the \"ssh\" program is not available but it is " - "required for downloading over the ssh protocol") + raise Error( + 'the "ssh" program is not available but it is ' + "required for downloading over the ssh protocol" + ) # Prepare the commands that we are going to run if password: # In case of password we have to use the sshpass tool to pass the # password to the ssh client utility - popen_args = ["sshpass", - "-p" + password, - "ssh", - "-o StrictHostKeyChecking=no", - "-o PubkeyAuthentication=no", - "-o PasswordAuthentication=yes", - hostname] + popen_args = [ + "sshpass", + "-p" + password, + "ssh", + "-o StrictHostKeyChecking=no", + "-o PubkeyAuthentication=no", + "-o PasswordAuthentication=yes", + hostname, + ] # Make sure the sshpass program is installed if not BmapHelpers.program_is_available("ssh"): - raise Error("the \"sshpass\" program is not available but it " - "is required for password-based SSH authentication") + raise Error( + 'the "sshpass" program is not available but it ' + "is required for password-based SSH authentication" + ) else: - popen_args = ["ssh", - "-o StrictHostKeyChecking=no", - "-o PubkeyAuthentication=yes", - "-o PasswordAuthentication=no", - "-o BatchMode=yes", - hostname] + popen_args = [ + "ssh", + "-o StrictHostKeyChecking=no", + "-o PubkeyAuthentication=yes", + "-o PasswordAuthentication=no", + "-o BatchMode=yes", + hostname, + ] # Test if we can successfully connect child_process = subprocess.Popen(popen_args + ["true"]) @@ -490,23 +528,27 @@ def _open_url_ssh(self, parsed_url): retcode = child_process.returncode if retcode != 0: decoded = _decode_sshpass_exit_code(retcode) - raise Error("cannot connect to \"%s\": %s (error code %d)" - % (hostname, decoded, retcode)) + raise Error( + 'cannot connect to "%s": %s (error code %d)' + % (hostname, decoded, retcode) + ) # Test if file exists by running "test -f path && test -r path" on the # host command = "test -f " + path + " && test -r " + path - child_process = subprocess.Popen(popen_args + [command], - bufsize=1024 * 1024) + child_process = subprocess.Popen(popen_args + [command], bufsize=1024 * 1024) child_process.wait() if child_process.returncode != 0: - raise Error("\"%s\" on \"%s\" cannot be read: make sure it " - "exists, is a regular file, and you have read " - "permissions" % (path, hostname)) + raise Error( + '"%s" on "%s" cannot be read: make sure it ' + "exists, is a regular file, and you have read " + "permissions" % (path, hostname) + ) # Read the entire file using 'cat' - child_process = subprocess.Popen(popen_args + ["cat " + path], - stdout=subprocess.PIPE) + child_process = subprocess.Popen( + popen_args + ["cat " + path], stdout=subprocess.PIPE + ) # Now the contents of the file should be available from sub-processes # stdout @@ -526,9 +568,10 @@ def _print_warning(timeout): This is a small helper function for printing a warning if we cannot open the URL for some time. """ - _log.warning("failed to open the URL with %d sec timeout, is the " - "proxy configured correctly? Keep trying ..." % - timeout) + _log.warning( + "failed to open the URL with %d sec timeout, is the " + "proxy configured correctly? Keep trying ..." % timeout + ) import socket @@ -565,7 +608,7 @@ def _print_warning(timeout): else: opener = urllib.build_opener() - opener.addheaders = [('User-Agent', 'Mozilla/5.0')] + opener.addheaders = [("User-Agent", "Mozilla/5.0")] urllib.install_opener(opener) # Open the URL. First try with a short timeout, and print a message @@ -587,16 +630,17 @@ def _print_warning(timeout): raise Error("cannot open URL '%s': %s" % (url, err)) except URLError as err: # Handling the timeout case in Python 2.6 - if timeout is not None and \ - isinstance(err.reason, socket.timeout): + if timeout is not None and isinstance(err.reason, socket.timeout): _print_warning(timeout) else: raise Error("cannot open URL '%s': %s" % (url, err)) except (IOError, ValueError, httplib.InvalidURL) as err: raise Error("cannot open URL '%s': %s" % (url, err)) except httplib.BadStatusLine: - raise Error("cannot open URL '%s': server responds with an " - "HTTP status code that we don't understand" % url) + raise Error( + "cannot open URL '%s': server responds with an " + "HTTP status code that we don't understand" % url + ) self.is_url = True self._f_objs.append(f_obj) @@ -617,15 +661,15 @@ def read(self, size=-1): def seek(self, offset, whence=os.SEEK_SET): """The 'seek()' method, similar to the one file objects have.""" if self._fake_seek or not hasattr(self._f_objs[-1], "seek"): - self._pos = _fake_seek_forward(self._f_objs[-1], self._pos, - offset, whence) + self._pos = _fake_seek_forward(self._f_objs[-1], self._pos, offset, whence) else: try: self._f_objs[-1].seek(offset, whence) except io.UnsupportedOperation: self._fake_seek = True - self._pos = _fake_seek_forward(self._f_objs[-1], self._pos, - offset, whence) + self._pos = _fake_seek_forward( + self._f_objs[-1], self._pos, offset, whence + ) def tell(self): """The 'tell()' method, similar to the one file objects have.""" @@ -644,7 +688,7 @@ def __getattr__(self, name): its operations. """ - if self.compression_type == 'none' and not self.is_url: + if self.compression_type == "none" and not self.is_url: return getattr(self._f_objs[-1], name) else: raise AttributeError diff --git a/contrib/bmap_write.py b/contrib/bmap_write.py index 41ba04c..14c97f9 100755 --- a/contrib/bmap_write.py +++ b/contrib/bmap_write.py @@ -39,22 +39,22 @@ print("raw-file '%s' doesn't exist" % raw_file) sys.exit(1) file_root, file_ext = os.path.splitext(raw_file) -bmap_file = file_root + '.bmap' +bmap_file = file_root + ".bmap" if not os.path.isfile(bmap_file): print("bmap-file '%s' doesn't exist" % bmap_file) sys.exit(1) bmap_root = ET.parse(bmap_file).getroot() -blocksize = int(bmap_root.find('BlockSize').text) -with open(raw_file, 'rb') as filedata: - with open(output_file, 'wb') as outdata: +blocksize = int(bmap_root.find("BlockSize").text) +with open(raw_file, "rb") as filedata: + with open(output_file, "wb") as outdata: try: - outdata.truncate(int(bmap_root.find('ImageSize').text)) # optional + outdata.truncate(int(bmap_root.find("ImageSize").text)) # optional except: pass - for bmap_range in bmap_root.find('BlockMap').findall('Range'): + for bmap_range in bmap_root.find("BlockMap").findall("Range"): blockrange = bmap_range.text - m = re.match('^\s*(\d+)\s*-\s*(\d+)\s*$', blockrange) + m = re.match("^\s*(\d+)\s*-\s*(\d+)\s*$", blockrange) if m: start = int(m.group(1)) end = int(m.group(2)) diff --git a/setup.py b/setup.py index c8bd22c..7acf5bd 100644 --- a/setup.py +++ b/setup.py @@ -14,29 +14,29 @@ def get_version(): return None + setup( name="bmap-tools", - description="Tools to generate block map (AKA bmap) and copy images " - "using bmap", + description="Tools to generate block map (AKA bmap) and copy images " "using bmap", author="Artem Bityutskiy", author_email="artem.bityutskiy@linux.intel.com", version=get_version(), entry_points={ - 'console_scripts': ['bmaptool=bmaptools.CLI:main'], + "console_scripts": ["bmaptool=bmaptools.CLI:main"], }, packages=find_packages(exclude=["test*"]), - license='GPLv2', + license="GPLv2", long_description="Tools to generate block map (AKA bmap) and flash " - "images using bmap. Bmaptool is a generic tool for " - "creating the block map (bmap) for a file, and copying " - "files using the block map. The idea is that large file " - "containing unused blocks, like raw system image files, " - "can be copied or flashed a lot faster with bmaptool " - "than with traditional tools like \"dd\" or \"cp\". See " - "source.tizen.org/documentation/reference/bmaptool for " - "more information.", + "images using bmap. Bmaptool is a generic tool for " + "creating the block map (bmap) for a file, and copying " + "files using the block map. The idea is that large file " + "containing unused blocks, like raw system image files, " + "can be copied or flashed a lot faster with bmaptool " + 'than with traditional tools like "dd" or "cp". See ' + "source.tizen.org/documentation/reference/bmaptool for " + "more information.", classifiers=[ "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3" - ] + "Programming Language :: Python :: 3", + ], ) diff --git a/tests/helpers.py b/tests/helpers.py index e3710b2..7160518 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -129,34 +129,34 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # # A block-sized hole - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Khole_", delete=delete, dir=directory, suffix=".img" + ) block_size = BmapHelpers.get_block_size(file_obj) file_obj.truncate(block_size) yield (file_obj, block_size, [], [(0, 0)]) file_obj.close() # A block size + 1 byte hole - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_plus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Khole_plus_1_", delete=delete, dir=directory, suffix=".img" + ) file_obj.truncate(block_size + 1) yield (file_obj, block_size + 1, [], [(0, 1)]) file_obj.close() # A block size - 1 byte hole - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_minus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Khole_minus_1_", delete=delete, dir=directory, suffix=".img" + ) file_obj.truncate(block_size - 1) yield (file_obj, block_size - 1, [], [(0, 0)]) file_obj.close() # A 1-byte hole - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_hole_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="1byte_hole_", delete=delete, dir=directory, suffix=".img" + ) file_obj.truncate(1) yield (file_obj, 1, [], [(0, 0)]) file_obj.close() @@ -164,9 +164,13 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # And 10 holes of random size for i in range(10): size = random.randint(1, max_size) - file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img", - delete=delete, dir=directory, - prefix="rand_hole_%d_" % i) + file_obj = tempfile.NamedTemporaryFile( + "wb+", + suffix=".img", + delete=delete, + dir=directory, + prefix="rand_hole_%d_" % i, + ) file_obj.truncate(size) blocks_cnt = (size + block_size - 1) // block_size yield (file_obj, size, [], [(0, blocks_cnt - 1)]) @@ -177,25 +181,25 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # # The maximum size - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="sparse_", delete=delete, dir=directory, suffix=".img" + ) mapped, unmapped = _create_random_sparse_file(file_obj, max_size) yield (file_obj, max_size, mapped, unmapped) file_obj.close() # The maximum size + 1 byte - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_plus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="sparse_plus_1_", delete=delete, dir=directory, suffix=".img" + ) mapped, unmapped = _create_random_sparse_file(file_obj, max_size + 1) yield (file_obj, max_size + 1, mapped, unmapped) file_obj.close() # The maximum size - 1 byte - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_minus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="sparse_minus_1_", delete=delete, dir=directory, suffix=".img" + ) mapped, unmapped = _create_random_sparse_file(file_obj, max_size - 1) yield (file_obj, max_size - 1, mapped, unmapped) file_obj.close() @@ -203,9 +207,9 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # And 10 files of random size for i in range(10): size = random.randint(1, max_size) - file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img", - delete=delete, dir=directory, - prefix="sparse_%d_" % i) + file_obj = tempfile.NamedTemporaryFile( + "wb+", suffix=".img", delete=delete, dir=directory, prefix="sparse_%d_" % i + ) mapped, unmapped = _create_random_sparse_file(file_obj, size) yield (file_obj, size, mapped, unmapped) file_obj.close() @@ -215,33 +219,33 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # # A block-sized file - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Kmapped_", delete=delete, dir=directory, suffix=".img" + ) _create_random_file(file_obj, block_size) yield (file_obj, block_size, [(0, 0)], []) file_obj.close() # A block size + 1 byte file - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_plus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Kmapped_plus_1_", delete=delete, dir=directory, suffix=".img" + ) _create_random_file(file_obj, block_size + 1) yield (file_obj, block_size + 1, [(0, 1)], []) file_obj.close() # A block size - 1 byte file - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_minus_1_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="4Kmapped_minus_1_", delete=delete, dir=directory, suffix=".img" + ) _create_random_file(file_obj, block_size - 1) yield (file_obj, block_size - 1, [(0, 0)], []) file_obj.close() # A 1-byte file - file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_mapped_", - delete=delete, dir=directory, - suffix=".img") + file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix="1byte_mapped_", delete=delete, dir=directory, suffix=".img" + ) _create_random_file(file_obj, 1) yield (file_obj, 1, [(0, 0)], []) file_obj.close() @@ -249,9 +253,13 @@ def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): # And 10 mapped files of random size for i in range(10): size = random.randint(1, max_size) - file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img", - delete=delete, dir=directory, - prefix="rand_mapped_%d_" % i) + file_obj = tempfile.NamedTemporaryFile( + "wb+", + suffix=".img", + delete=delete, + dir=directory, + prefix="rand_mapped_%d_" % i, + ) _create_random_file(file_obj, size) blocks_cnt = (size + block_size - 1) // block_size yield (file_obj, size, [(0, blocks_cnt - 1)], []) @@ -284,7 +292,7 @@ def copy_and_verify_image(image, dest, bmap, image_chksum, image_size): f_image = TransRead.TransRead(image) f_dest = open(dest, "w+b") - if (bmap): + if bmap: f_bmap = open(bmap, "r") else: f_bmap = None diff --git a/tests/oldcodebase/BmapCopy1_0.py b/tests/oldcodebase/BmapCopy1_0.py index ff1e6b4..8b10fdc 100644 --- a/tests/oldcodebase/BmapCopy1_0.py +++ b/tests/oldcodebase/BmapCopy1_0.py @@ -51,19 +51,22 @@ from bmaptools.BmapHelpers import human_size # A list of supported image formats -SUPPORTED_IMAGE_FORMATS = ('bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2') +SUPPORTED_IMAGE_FORMATS = ("bz2", "gz", "tar.gz", "tgz", "tar.bz2") # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -104,13 +107,13 @@ class BmapCopy: not. To explicitly synchronize it, use the 'sync()' method. This class supports all the bmap format versions up version - 'SUPPORTED_BMAP_VERSION'. """ + 'SUPPORTED_BMAP_VERSION'.""" def _initialize_sizes(self, image_size): - """ This function is only used when the there is no bmap. It + """This function is only used when the there is no bmap. It initializes attributes like 'blocks_cnt', 'mapped_cnt', etc. Normally, the values are read from the bmap file, but in this case they are just - set to something reasonable. """ + set to something reasonable.""" self.image_size = image_size self.image_size_human = human_size(image_size) @@ -120,7 +123,6 @@ def _initialize_sizes(self, image_size): self.mapped_size = self.image_size self.mapped_size_human = self.image_size_human - def _parse_bmap(self): """ Parse the bmap file and initialize the 'bmap_*' attributes. """ @@ -129,19 +131,22 @@ def _parse_bmap(self): try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - major = int(self.bmap_version.split('.', 1)[0]) + major = int(self.bmap_version.split(".", 1)[0]) if major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" % (SUPPORTED_BMAP_VERSION, major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -155,55 +160,63 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) self._f_bmap.seek(bmap_pos) def _open_image_file(self): - """ Open the image file which may be compressed or not. The compression + """Open the image file which may be compressed or not. The compression type is recognized by the file extension. Supported types are defined - by 'SUPPORTED_IMAGE_FORMATS'. """ + by 'SUPPORTED_IMAGE_FORMATS'.""" try: is_regular_file = stat.S_ISREG(os.stat(self._image_path).st_mode) except OSError as err: - raise Error("cannot access image file '%s': %s" \ - % (self._image_path, err.strerror)) + raise Error( + "cannot access image file '%s': %s" % (self._image_path, err.strerror) + ) if not is_regular_file: - raise Error("image file '%s' is not a regular file" \ - % self._image_path) + raise Error("image file '%s' is not a regular file" % self._image_path) try: - if self._image_path.endswith('.tar.gz') \ - or self._image_path.endswith('.tar.bz2') \ - or self._image_path.endswith('.tgz'): + if ( + self._image_path.endswith(".tar.gz") + or self._image_path.endswith(".tar.bz2") + or self._image_path.endswith(".tgz") + ): import tarfile - tar = tarfile.open(self._image_path, 'r') + tar = tarfile.open(self._image_path, "r") # The tarball is supposed to contain only one single member members = tar.getnames() if len(members) > 1: - raise Error("the image tarball '%s' contains more than " \ - "one file" % self._image_path) + raise Error( + "the image tarball '%s' contains more than " + "one file" % self._image_path + ) elif len(members) == 0: - raise Error("the image tarball '%s' is empty (no files)" \ - % self._image_path) + raise Error( + "the image tarball '%s' is empty (no files)" % self._image_path + ) self._f_image = tar.extractfile(members[0]) - elif self._image_path.endswith('.gz'): + elif self._image_path.endswith(".gz"): import gzip - self._f_image = gzip.GzipFile(self._image_path, 'rb') - elif self._image_path.endswith('.bz2'): + + self._f_image = gzip.GzipFile(self._image_path, "rb") + elif self._image_path.endswith(".bz2"): import bz2 - self._f_image = bz2.BZ2File(self._image_path, 'rb') + + self._f_image = bz2.BZ2File(self._image_path, "rb") else: self._image_is_compressed = False - self._f_image = open(self._image_path, 'rb') + self._f_image = open(self._image_path, "rb") except IOError as err: - raise Error("cannot open image file '%s': %s" \ - % (self._image_path, err)) + raise Error("cannot open image file '%s': %s" % (self._image_path, err)) self._f_image_needs_close = True @@ -212,19 +225,21 @@ def _validate_image_size(self): image_size = os.fstat(self._f_image.fileno()).st_size if image_size != self.image_size: - raise Error("Size mismatch, bmap '%s' was created for an image " \ - "of size %d bytes, but image '%s' has size %d bytes" \ - % (self._bmap_path, self.image_size, - self._image_path, image_size)) + raise Error( + "Size mismatch, bmap '%s' was created for an image " + "of size %d bytes, but image '%s' has size %d bytes" + % (self._bmap_path, self.image_size, self._image_path, image_size) + ) def _open_destination_file(self): """ Open the destination file. """ try: - self._f_dest = open(self._dest_path, 'w') + self._f_dest = open(self._dest_path, "w") except IOError as err: - raise Error("cannot open destination file '%s': %s" \ - % (self._dest_path, err)) + raise Error( + "cannot open destination file '%s': %s" % (self._dest_path, err) + ) self._f_dest_needs_close = True @@ -232,20 +247,21 @@ def _open_bmap_file(self): """ Open the bmap file. """ try: - self._f_bmap = open(self._bmap_path, 'r') + self._f_bmap = open(self._bmap_path, "r") except IOError as err: - raise Error("cannot open bmap file '%s': %s" \ - % (self._bmap_path, err.strerror)) + raise Error( + "cannot open bmap file '%s': %s" % (self._bmap_path, err.strerror) + ) self._f_bmap_needs_close = True - def __init__(self, image, dest, bmap = None): - """ The class constructor. The parameters are: - image - full path or file object of the image which should be copied - dest - full path or file-like object of the destination file to - copy the image to - bmap - full path or file-like object of the bmap file to use for - copying """ + def __init__(self, image, dest, bmap=None): + """The class constructor. The parameters are: + image - full path or file object of the image which should be copied + dest - full path or file-like object of the destination file to + copy the image to + bmap - full path or file-like object of the bmap file to use for + copying""" self._xml = None self._image_is_compressed = True @@ -327,7 +343,7 @@ def __del__(self): self._f_bmap.close() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -338,7 +354,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown (the image is compressed), the generator infinitely yields continuous ranges of - size '_batch_blocks'. """ + size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -362,7 +378,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -372,15 +388,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -389,7 +405,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -402,16 +418,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -420,9 +436,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -432,13 +449,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s" \ - % (first, last, hash_obj.hexdigest(), sha1)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s" + % (first, last, hash_obj.hexdigest(), sha1) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -449,11 +467,11 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The sync + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The sync argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Save file positions in order to restore them at the end image_pos = self._f_image.tell() @@ -464,7 +482,7 @@ def copy(self, sync = True, verify = True): # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -499,11 +517,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) if not self.image_size: @@ -515,16 +535,17 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks, but should have %u - inconsistent " \ - "bmap file" % (blocks_written, self.mapped_cnt)) + raise Error( + "wrote %u blocks, but should have %u - inconsistent " + "bmap file" % (blocks_written, self.mapped_cnt) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -541,21 +562,22 @@ def copy(self, sync = True, verify = True): self._f_bmap.seek(bmap_pos) def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _open_destination_file(self): """ Open the block device in exclusive mode. """ @@ -563,27 +585,29 @@ def _open_destination_file(self): try: self._f_dest = os.open(self._dest_path, os.O_WRONLY | os.O_EXCL) except OSError as err: - raise Error("cannot open block device '%s' in exclusive mode: %s" \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot open block device '%s' in exclusive mode: %s" + % (self._dest_path, err.strerror) + ) try: os.fstat(self._f_dest).st_mode except OSError as err: - raise Error("cannot access block device '%s': %s" \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot access block device '%s': %s" % (self._dest_path, err.strerror) + ) # Turn the block device file descriptor into a file object try: self._f_dest = os.fdopen(self._f_dest, "wb") except OSError as err: os.close(self._f_dest) - raise Error("cannot open block device '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot open block device '%s': %s" % (self._dest_path, err)) self._f_dest_needs_close = True def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -607,7 +631,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) self._old_scheduler_value = match.group(1) # Limit the write buffering @@ -620,8 +644,8 @@ def _tune_block_device(self): return def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: @@ -638,8 +662,8 @@ def _restore_bdev_settings(self): except IOError: return - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -648,7 +672,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" try: self._tune_block_device() @@ -657,9 +681,9 @@ def copy(self, sync = True, verify = True): self._restore_bdev_settings() raise - def __init__(self, image, dest, bmap = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap) @@ -682,19 +706,29 @@ def __init__(self, image, dest, bmap = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_0.py b/tests/oldcodebase/BmapCopy2_0.py index f376af1..48b4529 100644 --- a/tests/oldcodebase/BmapCopy2_0.py +++ b/tests/oldcodebase/BmapCopy2_0.py @@ -50,14 +50,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -100,10 +103,10 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance.""" def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been + """Setup the progress indicator which shows how much data has been copied in percent. The 'file_obj' argument is the console file object where the progress @@ -111,7 +114,7 @@ def set_progress_indicator(self, file_obj, format_string): The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent.""" self._progress_file = file_obj if format_string: @@ -120,13 +123,15 @@ def set_progress_indicator(self, file_obj, format_string): self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """Set image size and initialize various other geometry-related + attributes.""" if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " \ - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -139,24 +144,27 @@ def _set_image_size(self, image_size): self.mapped_size_human = self.image_size_human def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """Parse the bmap file and initialize corresponding class instance + attributs.""" try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - major = int(self.bmap_version.split('.', 1)[0]) + major = int(self.bmap_version.split(".", 1)[0]) if major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" % (SUPPORTED_BMAP_VERSION, major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -170,19 +178,21 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) - - def __init__(self, image, dest, bmap = None, image_size = None): - """ The class constructor. The parameters are: - image - file-like object of the image which should be copied, - should only support 'read()' and 'seek()' methods, - and only seeking forward has to be supported. - dest - file-like object of the destination file to copy the - image to. - bmap - file-like object of the bmap file to use for copying. - image_size - size of the image in bytes. """ + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) + + def __init__(self, image, dest, bmap=None, image_size=None): + """The class constructor. The parameters are: + image - file-like object of the image which should be copied, + should only support 'read()' and 'seek()' methods, + and only seeking forward has to be supported. + dest - file-like object of the destination file to copy the + image to. + bmap - file-like object of the bmap file to use for copying. + image_size - size of the image in bytes.""" self._xml = None @@ -221,9 +231,11 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -245,9 +257,9 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._batch_blocks = self._batch_bytes / self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and + """Print the progress indicator if the mapped area size is known and if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + the 'progress_file' attribute.""" if not self._progress_file: return @@ -255,17 +267,17 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() - min_delta = datetime.timedelta(milliseconds = 250) + min_delta = datetime.timedelta(milliseconds=250) if now - self._progress_time < min_delta: return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -275,7 +287,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') + self._progress_file.write("\033[1A") else: self._progress_started = True @@ -283,7 +295,7 @@ def _update_progress(self, blocks_written): self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -293,7 +305,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -317,7 +329,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -327,15 +339,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -344,7 +356,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -357,16 +369,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -375,9 +387,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -387,14 +400,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s (image file %s)" \ - % (first, last, hash_obj.hexdigest(), \ - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -405,16 +418,16 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The 'sync' argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -453,11 +466,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -469,18 +484,24 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " \ - "have %u - inconsistent bmap file '%s'" \ - % (blocks_written, self._image_path, self._dest_path, \ - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - inconsistent bmap file '%s'" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -491,25 +512,26 @@ def copy(self, sync = True, verify = True): self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" if self._dest_supports_fsync: try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -533,7 +555,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) self._old_scheduler_value = match.group(1) # Limit the write buffering @@ -546,27 +568,31 @@ def _tune_block_device(self): raise Error("cannot set max. I/O ratio to '1': %s" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" \ - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" \ - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -575,7 +601,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" self._tune_block_device() @@ -586,9 +612,9 @@ def copy(self, sync = True, verify = True): finally: self._restore_bdev_settings() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size) @@ -610,19 +636,29 @@ def __init__(self, image, dest, bmap = None, image_size = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_1.py b/tests/oldcodebase/BmapCopy2_1.py index ee75ae8..e81d7aa 100644 --- a/tests/oldcodebase/BmapCopy2_1.py +++ b/tests/oldcodebase/BmapCopy2_1.py @@ -50,14 +50,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -100,10 +103,10 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance.""" def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been + """Setup the progress indicator which shows how much data has been copied in percent. The 'file_obj' argument is the console file object where the progress @@ -111,7 +114,7 @@ def set_progress_indicator(self, file_obj, format_string): The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent.""" self._progress_file = file_obj if format_string: @@ -120,13 +123,15 @@ def set_progress_indicator(self, file_obj, format_string): self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """Set image size and initialize various other geometry-related + attributes.""" if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " \ - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -138,24 +143,27 @@ def _set_image_size(self, image_size): self.mapped_size_human = self.image_size_human def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """Parse the bmap file and initialize corresponding class instance + attributs.""" try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - major = int(self.bmap_version.split('.', 1)[0]) + major = int(self.bmap_version.split(".", 1)[0]) if major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" % (SUPPORTED_BMAP_VERSION, major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -169,19 +177,21 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) - - def __init__(self, image, dest, bmap = None, image_size = None): - """ The class constructor. The parameters are: - image - file-like object of the image which should be copied, - should only support 'read()' and 'seek()' methods, - and only seeking forward has to be supported. - dest - file-like object of the destination file to copy the - image to. - bmap - file-like object of the bmap file to use for copying. - image_size - size of the image in bytes. """ + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) + + def __init__(self, image, dest, bmap=None, image_size=None): + """The class constructor. The parameters are: + image - file-like object of the image which should be copied, + should only support 'read()' and 'seek()' methods, + and only seeking forward has to be supported. + dest - file-like object of the destination file to copy the + image to. + bmap - file-like object of the bmap file to use for copying. + image_size - size of the image in bytes.""" self._xml = None @@ -220,9 +230,11 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -244,9 +256,9 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._batch_blocks = self._batch_bytes // self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and + """Print the progress indicator if the mapped area size is known and if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + the 'progress_file' attribute.""" if not self._progress_file: return @@ -254,17 +266,17 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() - min_delta = datetime.timedelta(milliseconds = 250) + min_delta = datetime.timedelta(milliseconds=250) if now - self._progress_time < min_delta: return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -274,7 +286,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') + self._progress_file.write("\033[1A") else: self._progress_started = True @@ -282,7 +294,7 @@ def _update_progress(self, blocks_written): self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -292,7 +304,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -316,7 +328,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -326,15 +338,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -343,7 +355,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -356,16 +368,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -374,9 +386,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -386,14 +399,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) // self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s (image file %s)" \ - % (first, last, hash_obj.hexdigest(), \ - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -404,16 +417,16 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The 'sync' argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -452,11 +465,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -468,18 +483,24 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " \ - "have %u - inconsistent bmap file '%s'" \ - % (blocks_written, self._image_path, self._dest_path, \ - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - inconsistent bmap file '%s'" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -490,25 +511,26 @@ def copy(self, sync = True, verify = True): self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" if self._dest_supports_fsync: try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -532,7 +554,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) self._old_scheduler_value = match.group(1) # Limit the write buffering @@ -545,27 +567,31 @@ def _tune_block_device(self): raise Error("cannot set max. I/O ratio to '1': %s" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" \ - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" \ - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -574,7 +600,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" self._tune_block_device() @@ -585,9 +611,9 @@ def copy(self, sync = True, verify = True): finally: self._restore_bdev_settings() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size) @@ -609,19 +635,29 @@ def __init__(self, image, dest, bmap = None, image_size = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_2.py b/tests/oldcodebase/BmapCopy2_2.py index b5f537c..75aa7f3 100644 --- a/tests/oldcodebase/BmapCopy2_2.py +++ b/tests/oldcodebase/BmapCopy2_2.py @@ -50,14 +50,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -100,10 +103,10 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance.""" def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been + """Setup the progress indicator which shows how much data has been copied in percent. The 'file_obj' argument is the console file object where the progress @@ -111,7 +114,7 @@ def set_progress_indicator(self, file_obj, format_string): The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent.""" self._progress_file = file_obj if format_string: @@ -120,13 +123,15 @@ def set_progress_indicator(self, file_obj, format_string): self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """Set image size and initialize various other geometry-related + attributes.""" if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " \ - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -139,24 +144,27 @@ def _set_image_size(self, image_size): self.mapped_size_human = self.image_size_human def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """Parse the bmap file and initialize corresponding class instance + attributs.""" try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - major = int(self.bmap_version.split('.', 1)[0]) + major = int(self.bmap_version.split(".", 1)[0]) if major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" % (SUPPORTED_BMAP_VERSION, major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -170,19 +178,21 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) - - def __init__(self, image, dest, bmap = None, image_size = None): - """ The class constructor. The parameters are: - image - file-like object of the image which should be copied, - should only support 'read()' and 'seek()' methods, - and only seeking forward has to be supported. - dest - file-like object of the destination file to copy the - image to. - bmap - file-like object of the bmap file to use for copying. - image_size - size of the image in bytes. """ + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) + + def __init__(self, image, dest, bmap=None, image_size=None): + """The class constructor. The parameters are: + image - file-like object of the image which should be copied, + should only support 'read()' and 'seek()' methods, + and only seeking forward has to be supported. + dest - file-like object of the destination file to copy the + image to. + bmap - file-like object of the bmap file to use for copying. + image_size - size of the image in bytes.""" self._xml = None @@ -221,9 +231,11 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -245,9 +257,9 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._batch_blocks = self._batch_bytes / self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and + """Print the progress indicator if the mapped area size is known and if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + the 'progress_file' attribute.""" if not self._progress_file: return @@ -255,17 +267,17 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() - min_delta = datetime.timedelta(milliseconds = 250) + min_delta = datetime.timedelta(milliseconds=250) if now - self._progress_time < min_delta: return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -275,7 +287,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -283,7 +295,7 @@ def _update_progress(self, blocks_written): self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -293,7 +305,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -317,7 +329,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -327,15 +339,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -344,7 +356,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -357,16 +369,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -375,9 +387,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -387,14 +400,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s (image file %s)" \ - % (first, last, hash_obj.hexdigest(), \ - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -405,16 +418,16 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The 'sync' argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -453,11 +466,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -469,18 +484,24 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " \ - "have %u - inconsistent bmap file '%s'" \ - % (blocks_written, self._image_path, self._dest_path, \ - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - inconsistent bmap file '%s'" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -491,25 +512,26 @@ def copy(self, sync = True, verify = True): self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" if self._dest_supports_fsync: try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -533,7 +555,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -547,27 +569,31 @@ def _tune_block_device(self): raise Error("cannot set max. I/O ratio to '1': %s" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" \ - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" \ - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -576,7 +602,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" self._tune_block_device() @@ -587,9 +613,9 @@ def copy(self, sync = True, verify = True): finally: self._restore_bdev_settings() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size) @@ -611,19 +637,29 @@ def __init__(self, image, dest, bmap = None, image_size = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_3.py b/tests/oldcodebase/BmapCopy2_3.py index 18393af..28959b8 100644 --- a/tests/oldcodebase/BmapCopy2_3.py +++ b/tests/oldcodebase/BmapCopy2_3.py @@ -50,14 +50,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -100,10 +103,10 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance.""" def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been + """Setup the progress indicator which shows how much data has been copied in percent. The 'file_obj' argument is the console file object where the progress @@ -111,7 +114,7 @@ def set_progress_indicator(self, file_obj, format_string): The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent.""" self._progress_file = file_obj if format_string: @@ -120,13 +123,15 @@ def set_progress_indicator(self, file_obj, format_string): self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """Set image size and initialize various other geometry-related + attributes.""" if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " \ - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -139,8 +144,8 @@ def _set_image_size(self, image_size): self.mapped_size_human = self.image_size_human def _verify_bmap_checksum(self): - """ This is a helper function which verifies SHA1 checksum of the bmap - file. """ + """This is a helper function which verifies SHA1 checksum of the bmap + file.""" import mmap @@ -149,42 +154,47 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the SHA1 value # stored in the file with all zeroes. For these purposes we create # private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access = mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) sha1_pos = mapped_bmap.find(correct_sha1) assert sha1_pos != -1 - mapped_bmap[sha1_pos:sha1_pos + 40] = '0' * 40 + mapped_bmap[sha1_pos : sha1_pos + 40] = "0" * 40 calculated_sha1 = hashlib.sha1(mapped_bmap).hexdigest() mapped_bmap.close() if calculated_sha1 != correct_sha1: - raise Error("checksum mismatch for bmap file '%s': calculated " \ - "'%s', should be '%s'" % \ - (self._bmap_path, calculated_sha1, correct_sha1)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_sha1, correct_sha1) + ) def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """Parse the bmap file and initialize corresponding class instance + attributs.""" try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -198,23 +208,25 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 self._verify_bmap_checksum() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The class constructor. The parameters are: - image - file-like object of the image which should be copied, - should only support 'read()' and 'seek()' methods, - and only seeking forward has to be supported. - dest - file object of the destination file to copy the image - to. - bmap - file object of the bmap file to use for copying. - image_size - size of the image in bytes. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The class constructor. The parameters are: + image - file-like object of the image which should be copied, + should only support 'read()' and 'seek()' methods, + and only seeking forward has to be supported. + dest - file object of the destination file to copy the image + to. + bmap - file object of the bmap file to use for copying. + image_size - size of the image in bytes.""" self._xml = None @@ -255,9 +267,11 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -279,9 +293,9 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._batch_blocks = self._batch_bytes / self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and + """Print the progress indicator if the mapped area size is known and if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + the 'progress_file' attribute.""" if not self._progress_file: return @@ -289,17 +303,17 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() - min_delta = datetime.timedelta(milliseconds = 250) + min_delta = datetime.timedelta(milliseconds=250) if now - self._progress_time < min_delta: return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -309,7 +323,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -317,7 +331,7 @@ def _update_progress(self, blocks_written): self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -327,7 +341,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -351,7 +365,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -361,15 +375,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -378,7 +392,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -391,16 +405,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -409,9 +423,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -421,14 +436,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s (image file %s)" \ - % (first, last, hash_obj.hexdigest(), \ - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -439,16 +454,16 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The 'sync' argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -487,11 +502,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -503,19 +520,25 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " \ - "have %u - bmap file '%s' does not belong to this" \ - "image" \ - % (blocks_written, self._image_path, self._dest_path, \ - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this" + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -526,25 +549,26 @@ def copy(self, sync = True, verify = True): self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" if self._dest_supports_fsync: try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -568,7 +592,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -582,27 +606,31 @@ def _tune_block_device(self): raise Error("cannot set max. I/O ratio to '1': %s" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" \ - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" \ - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -611,7 +639,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" self._tune_block_device() @@ -622,9 +650,9 @@ def copy(self, sync = True, verify = True): finally: self._restore_bdev_settings() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size) @@ -646,19 +674,29 @@ def __init__(self, image, dest, bmap = None, image_size = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_4.py b/tests/oldcodebase/BmapCopy2_4.py index 18393af..28959b8 100644 --- a/tests/oldcodebase/BmapCopy2_4.py +++ b/tests/oldcodebase/BmapCopy2_4.py @@ -50,14 +50,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors.""" + pass + class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -100,10 +103,10 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance.""" def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been + """Setup the progress indicator which shows how much data has been copied in percent. The 'file_obj' argument is the console file object where the progress @@ -111,7 +114,7 @@ def set_progress_indicator(self, file_obj, format_string): The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent.""" self._progress_file = file_obj if format_string: @@ -120,13 +123,15 @@ def set_progress_indicator(self, file_obj, format_string): self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """Set image size and initialize various other geometry-related + attributes.""" if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " \ - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -139,8 +144,8 @@ def _set_image_size(self, image_size): self.mapped_size_human = self.image_size_human def _verify_bmap_checksum(self): - """ This is a helper function which verifies SHA1 checksum of the bmap - file. """ + """This is a helper function which verifies SHA1 checksum of the bmap + file.""" import mmap @@ -149,42 +154,47 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the SHA1 value # stored in the file with all zeroes. For these purposes we create # private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access = mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) sha1_pos = mapped_bmap.find(correct_sha1) assert sha1_pos != -1 - mapped_bmap[sha1_pos:sha1_pos + 40] = '0' * 40 + mapped_bmap[sha1_pos : sha1_pos + 40] = "0" * 40 calculated_sha1 = hashlib.sha1(mapped_bmap).hexdigest() mapped_bmap.close() if calculated_sha1 != correct_sha1: - raise Error("checksum mismatch for bmap file '%s': calculated " \ - "'%s', should be '%s'" % \ - (self._bmap_path, calculated_sha1, correct_sha1)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_sha1, correct_sha1) + ) def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """Parse the bmap file and initialize corresponding class instance + attributs.""" try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " \ - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " \ - "version %d is not supported" \ - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -198,23 +208,25 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " \ - "blocks count (%d bytes != %d blocks * %d bytes)" \ - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 self._verify_bmap_checksum() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The class constructor. The parameters are: - image - file-like object of the image which should be copied, - should only support 'read()' and 'seek()' methods, - and only seeking forward has to be supported. - dest - file object of the destination file to copy the image - to. - bmap - file object of the bmap file to use for copying. - image_size - size of the image in bytes. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The class constructor. The parameters are: + image - file-like object of the image which should be copied, + should only support 'read()' and 'seek()' methods, + and only seeking forward has to be supported. + dest - file object of the destination file to copy the image + to. + bmap - file object of the bmap file to use for copying. + image_size - size of the image in bytes.""" self._xml = None @@ -255,9 +267,11 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -279,9 +293,9 @@ def __init__(self, image, dest, bmap = None, image_size = None): self._batch_blocks = self._batch_bytes / self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and + """Print the progress indicator if the mapped area size is known and if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + the 'progress_file' attribute.""" if not self._progress_file: return @@ -289,17 +303,17 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() - min_delta = datetime.timedelta(milliseconds = 250) + min_delta = datetime.timedelta(milliseconds=250) if now - self._progress_time < min_delta: return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -309,7 +323,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -317,7 +331,7 @@ def _update_progress(self, blocks_written): self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for + """This is a helper generator that parses the bmap XML file and for each block range in the XML file it yields ('first', 'last', 'sha1') tuples, where: * 'first' is the first block of the range; @@ -327,7 +341,7 @@ def _get_block_ranges(self): If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'.""" if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -351,7 +365,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -361,15 +375,15 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap + """This is a helper generator which splits block ranges from the bmap file to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the @@ -378,7 +392,7 @@ def _get_batches(self, first, last): * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1).""" batch_blocks = self._batch_blocks @@ -391,16 +405,16 @@ def _get_batches(self, first, last): yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' + """This is generator which reads the image file in '_batch_blocks' chunks and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data.""" try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -409,9 +423,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " \ - "image file '%s': %s" \ - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -421,14 +436,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " \ - "calculated %s, should be %s (image file %s)" \ - % (first, last, hash_obj.hexdigest(), \ - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -439,16 +454,16 @@ def _get_data(self, verify): self._batch_queue.put(None) - def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' + def copy(self, sync=True, verify=True): + """Copy the image to the destination file using bmap. The 'sync' argument defines whether the destination file has to be synchronized upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + has to be verified while copying.""" # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -487,11 +502,13 @@ def copy(self, sync = True, verify = True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" \ - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -503,19 +520,25 @@ def copy(self, sync = True, verify = True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " \ - "have %u - bmap file '%s' does not belong to this" \ - "image" \ - % (blocks_written, self._image_path, self._dest_path, \ - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this" + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" \ - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -526,25 +549,26 @@ def copy(self, sync = True, verify = True): self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """Synchronize the destination file to make sure all the data are + actually written to the disk.""" if self._dest_supports_fsync: try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the + """This class is a specialized version of 'BmapCopy' which copies the image to a block device. Unlike the base 'BmapCopy' class, this class does various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + 'noop' I/O scheduler.""" def _tune_block_device(self): - """" Tune the block device for better performance: + """ " Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -568,7 +592,7 @@ def _tune_block_device(self): # Fetch the current scheduler name import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -582,27 +606,31 @@ def _tune_block_device(self): raise Error("cannot set max. I/O ratio to '1': %s" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """Restore old block device settings which we changed in + '_tune_block_device()'.""" if self._old_scheduler_value is not None: try: with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" \ - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" \ - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) - def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better + def copy(self, sync=True, verify=True): + """The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the @@ -611,7 +639,7 @@ def copy(self, sync = True, verify = True): the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time.""" self._tune_block_device() @@ -622,9 +650,9 @@ def copy(self, sync = True, verify = True): finally: self._restore_bdev_settings() - def __init__(self, image, dest, bmap = None, image_size = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + def __init__(self, image, dest, bmap=None, image_size=None): + """The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices.""" # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size) @@ -646,19 +674,29 @@ def __init__(self, image, dest, bmap = None, image_size = None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " \ - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " \ - "fit the block device '%s' which has %s capacity" \ - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_5.py b/tests/oldcodebase/BmapCopy2_5.py index da6bab6..aa7e424 100644 --- a/tests/oldcodebase/BmapCopy2_5.py +++ b/tests/oldcodebase/BmapCopy2_5.py @@ -64,14 +64,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): """ A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable problem description in case of errors. """ + pass + class BmapCopy: """ This class implements the bmap-based copying functionality. To copy an @@ -145,9 +148,11 @@ def _set_image_size(self, image_size): """ if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -171,21 +176,22 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the SHA1 value # stored in the file with all zeroes. For these purposes we create # private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access = mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) sha1_pos = mapped_bmap.find(correct_sha1) assert sha1_pos != -1 - mapped_bmap[sha1_pos:sha1_pos + 40] = '0' * 40 + mapped_bmap[sha1_pos : sha1_pos + 40] = "0" * 40 calculated_sha1 = hashlib.sha1(mapped_bmap).hexdigest() mapped_bmap.close() if calculated_sha1 != correct_sha1: - raise Error("checksum mismatch for bmap file '%s': calculated " - "'%s', should be '%s'" - % (self._bmap_path, calculated_sha1, correct_sha1)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_sha1, correct_sha1) + ) def _parse_bmap(self): """ @@ -194,20 +200,24 @@ def _parse_bmap(self): try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " - "version %d is not supported" - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -221,9 +231,11 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " - "blocks count (%d bytes != %d blocks * %d bytes)" - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 @@ -285,9 +297,11 @@ def __init__(self, image, dest, bmap=None, image_size=None, logger=None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -321,7 +335,7 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() @@ -330,8 +344,8 @@ def _update_progress(self, blocks_written): return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -341,7 +355,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -385,7 +399,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -395,8 +409,8 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None @@ -438,7 +452,7 @@ def _get_data(self, verify): try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -447,9 +461,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " - "image file '%s': %s" - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -459,14 +474,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " - "calculated %s, should be %s (image file %s)" - % (first, last, hash_obj.hexdigest(), - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -488,7 +503,7 @@ def copy(self, sync=True, verify=True): # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -527,11 +542,13 @@ def copy(self, sync=True, verify=True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -543,19 +560,25 @@ def copy(self, sync=True, verify=True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " - "have %u - bmap file '%s' does not belong to this" - "image" - % (blocks_written, self._image_path, self._dest_path, - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this" + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -575,8 +598,9 @@ def sync(self): try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): @@ -605,16 +629,18 @@ def _tune_block_device(self): f_scheduler.seek(0) f_scheduler.write("noop") except IOError as err: - self._logger.warning("failed to enable I/O optimization, expect " - "suboptimal speed (reason: cannot switch " - "to the 'noop' I/O scheduler: %s)" % err) + self._logger.warning( + "failed to enable I/O optimization, expect " + "suboptimal speed (reason: cannot switch " + "to the 'noop' I/O scheduler: %s)" % err + ) else: # The file contains a list of schedulers with the current # scheduler in square brackets, e.g., "noop deadline [cfq]". # Fetch the name of the current scheduler. import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -627,10 +653,12 @@ def _tune_block_device(self): f_ratio.seek(0) f_ratio.write("1") except IOError as err: - self._logger.warning("failed to disable excessive buffering, " - "expect worse system responsiveness " - "(reason: cannot set max. I/O ratio to " - "1: %s)" % err) + self._logger.warning( + "failed to disable excessive buffering, " + "expect worse system responsiveness " + "(reason: cannot set max. I/O ratio to " + "1: %s)" % err + ) def _restore_bdev_settings(self): """ @@ -643,16 +671,20 @@ def _restore_bdev_settings(self): with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) def copy(self, sync=True, verify=True): """ @@ -703,19 +735,29 @@ def __init__(self, image, dest, bmap=None, image_size=None, logger=None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " - "fit the block device '%s' which has %s capacity" - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we diff --git a/tests/oldcodebase/BmapCopy2_6.py b/tests/oldcodebase/BmapCopy2_6.py index d08dd51..7276c0f 100644 --- a/tests/oldcodebase/BmapCopy2_6.py +++ b/tests/oldcodebase/BmapCopy2_6.py @@ -64,14 +64,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): """ A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable problem description in case of errors. """ + pass + class BmapCopy: """ This class implements the bmap-based copying functionality. To copy an @@ -176,9 +179,11 @@ def __init__(self, image, dest, bmap=None, image_size=None, logger=None): self._dest_is_regfile = stat.S_ISREG(st_data.st_mode) # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -224,9 +229,11 @@ def _set_image_size(self, image_size): """ if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -250,21 +257,22 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the SHA1 value # stored in the file with all zeroes. For these purposes we create # private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access = mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) sha1_pos = mapped_bmap.find(correct_sha1) assert sha1_pos != -1 - mapped_bmap[sha1_pos:sha1_pos + 40] = '0' * 40 + mapped_bmap[sha1_pos : sha1_pos + 40] = "0" * 40 calculated_sha1 = hashlib.sha1(mapped_bmap).hexdigest() mapped_bmap.close() if calculated_sha1 != correct_sha1: - raise Error("checksum mismatch for bmap file '%s': calculated " - "'%s', should be '%s'" - % (self._bmap_path, calculated_sha1, correct_sha1)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_sha1, correct_sha1) + ) def _parse_bmap(self): """ @@ -273,20 +281,24 @@ def _parse_bmap(self): try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: - raise Error("cannot parse the bmap file '%s' which should be a " - "proper XML file: %s" % (self._bmap_path, err)) + except ElementTree.ParseError as err: + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s" % (self._bmap_path, err) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " - "version %d is not supported" - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -300,9 +312,11 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " - "blocks count (%d bytes != %d blocks * %d bytes)" - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 @@ -321,7 +335,7 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() @@ -330,8 +344,8 @@ def _update_progress(self, blocks_written): return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -341,7 +355,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -385,7 +399,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -395,8 +409,8 @@ def _get_block_ranges(self): else: last = first - if 'sha1' in xml_element.attrib: - sha1 = xml_element.attrib['sha1'] + if "sha1" in xml_element.attrib: + sha1 = xml_element.attrib["sha1"] else: sha1 = None @@ -438,7 +452,7 @@ def _get_data(self, verify): try: for (first, last, sha1) in self._get_block_ranges(): if verify and sha1: - hash_obj = hashlib.new('sha1') + hash_obj = hashlib.new("sha1") self._f_image.seek(first * self.block_size) @@ -447,9 +461,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " - "image file '%s': %s" - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -459,14 +474,14 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and sha1 and hash_obj.hexdigest() != sha1: - raise Error("checksum mismatch for blocks range %d-%d: " - "calculated %s, should be %s (image file %s)" - % (first, last, hash_obj.hexdigest(), - sha1, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), sha1, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -488,7 +503,7 @@ def copy(self, sync=True, verify=True): # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -527,11 +542,13 @@ def copy(self, sync=True, verify=True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -543,19 +560,25 @@ def copy(self, sync=True, verify=True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " - "have %u - bmap file '%s' does not belong to this " - "image" - % (blocks_written, self._image_path, self._dest_path, - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this " + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -575,8 +598,9 @@ def sync(self): try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): @@ -613,19 +637,29 @@ def __init__(self, image, dest, bmap=None, image_size=None, logger=None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " - "fit the block device '%s' which has %s capacity" - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" \ - % (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we @@ -654,16 +688,18 @@ def _tune_block_device(self): f_scheduler.seek(0) f_scheduler.write("noop") except IOError as err: - self._logger.warning("failed to enable I/O optimization, expect " - "suboptimal speed (reason: cannot switch " - "to the 'noop' I/O scheduler: %s)" % err) + self._logger.warning( + "failed to enable I/O optimization, expect " + "suboptimal speed (reason: cannot switch " + "to the 'noop' I/O scheduler: %s)" % err + ) else: # The file contains a list of schedulers with the current # scheduler in square brackets, e.g., "noop deadline [cfq]". # Fetch the name of the current scheduler. import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -676,10 +712,12 @@ def _tune_block_device(self): f_ratio.seek(0) f_ratio.write("1") except IOError as err: - self._logger.warning("failed to disable excessive buffering, " - "expect worse system responsiveness " - "(reason: cannot set max. I/O ratio to " - "1: %s)" % err) + self._logger.warning( + "failed to disable excessive buffering, " + "expect worse system responsiveness " + "(reason: cannot set max. I/O ratio to " + "1: %s)" % err + ) def _restore_bdev_settings(self): """ @@ -692,16 +730,20 @@ def _restore_bdev_settings(self): with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) def copy(self, sync=True, verify=True): """ diff --git a/tests/oldcodebase/BmapCopy3_0.py b/tests/oldcodebase/BmapCopy3_0.py index 51c9460..2ccd126 100644 --- a/tests/oldcodebase/BmapCopy3_0.py +++ b/tests/oldcodebase/BmapCopy3_0.py @@ -64,14 +64,17 @@ # The highest supported bmap format version SUPPORTED_BMAP_VERSION = "1.0" + class Error(Exception): """ A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable problem description in case of errors. """ + pass + class BmapCopy: """ This class implements the bmap-based copying functionality. To copy an @@ -181,9 +184,11 @@ def __init__(self, image, dest, bmap=None, image_size=None, log=None): self._cs_attrib_name = None # Special quirk for /dev/null which does not support fsync() - if stat.S_ISCHR(st_data.st_mode) and \ - os.major(st_data.st_rdev) == 1 and \ - os.minor(st_data.st_rdev) == 3: + if ( + stat.S_ISCHR(st_data.st_mode) + and os.major(st_data.st_rdev) == 1 + and os.minor(st_data.st_rdev) == 3 + ): self._dest_supports_fsync = False else: self._dest_supports_fsync = True @@ -229,9 +234,11 @@ def _set_image_size(self, image_size): """ if self.image_size is not None and self.image_size != image_size: - raise Error("cannot set image size to %d bytes, it is known to " - "be %d bytes (%s)" % (image_size, self.image_size, - self.image_size_human)) + raise Error( + "cannot set image size to %d bytes, it is known to " + "be %d bytes (%s)" + % (image_size, self.image_size, self.image_size_human) + ) self.image_size = image_size self.image_size_human = human_size(image_size) @@ -258,13 +265,12 @@ def _verify_bmap_checksum(self): # Before verifying the shecksum, we have to substitute the checksum # value stored in the file with all zeroes. For these purposes we # create private memory mapping of the bmap file. - mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, - access = mmap.ACCESS_COPY) + mapped_bmap = mmap.mmap(self._f_bmap.fileno(), 0, access=mmap.ACCESS_COPY) chksum_pos = mapped_bmap.find(correct_chksum) assert chksum_pos != -1 - mapped_bmap[chksum_pos:chksum_pos + self._cs_len] = '0' * self._cs_len + mapped_bmap[chksum_pos : chksum_pos + self._cs_len] = "0" * self._cs_len hash_obj = hashlib.new(self._cs_type) hash_obj.update(mapped_bmap) @@ -273,9 +279,11 @@ def _verify_bmap_checksum(self): mapped_bmap.close() if calculated_chksum != correct_chksum: - raise Error("checksum mismatch for bmap file '%s': calculated " - "'%s', should be '%s'" - % (self._bmap_path, calculated_chksum, correct_chksum)) + raise Error( + "checksum mismatch for bmap file '%s': calculated " + "'%s', should be '%s'" + % (self._bmap_path, calculated_chksum, correct_chksum) + ) def _parse_bmap(self): """ @@ -284,7 +292,7 @@ def _parse_bmap(self): try: self._xml = ElementTree.parse(self._f_bmap) - except ElementTree.ParseError as err: + except ElementTree.ParseError as err: # Extrace the erroneous line with some context self._f_bmap.seek(0) xml_extract = "" @@ -292,20 +300,24 @@ def _parse_bmap(self): if num >= err.position[0] - 4 and num <= err.position[0] + 4: xml_extract += "Line %d: %s" % (num, line) - raise Error("cannot parse the bmap file '%s' which should be a " - "proper XML file: %s, the XML extract:\n%s" % - (self._bmap_path, err, xml_extract)) + raise Error( + "cannot parse the bmap file '%s' which should be a " + "proper XML file: %s, the XML extract:\n%s" + % (self._bmap_path, err, xml_extract) + ) xml = self._xml - self.bmap_version = str(xml.getroot().attrib.get('version')) + self.bmap_version = str(xml.getroot().attrib.get("version")) # Make sure we support this version - self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) - self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) + self.bmap_version_major = int(self.bmap_version.split(".", 1)[0]) + self.bmap_version_minor = int(self.bmap_version.split(".", 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: - raise Error("only bmap format version up to %d is supported, " - "version %d is not supported" - % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) + raise Error( + "only bmap format version up to %d is supported, " + "version %d is not supported" + % (SUPPORTED_BMAP_VERSION, self.bmap_version_major) + ) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) @@ -319,9 +331,11 @@ def _parse_bmap(self): blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: - raise Error("Inconsistent bmap - image size does not match " - "blocks count (%d bytes != %d blocks * %d bytes)" - % (self.image_size, self.blocks_cnt, self.block_size)) + raise Error( + "Inconsistent bmap - image size does not match " + "blocks count (%d bytes != %d blocks * %d bytes)" + % (self.image_size, self.blocks_cnt, self.block_size) + ) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 and the only supported @@ -339,8 +353,9 @@ def _parse_bmap(self): try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: - raise Error("cannot initialize hash function \"%s\": %s" % - (self._cs_type, err)) + raise Error( + 'cannot initialize hash function "%s": %s' % (self._cs_type, err) + ) self._verify_bmap_checksum() def _update_progress(self, blocks_written): @@ -353,8 +368,10 @@ def _update_progress(self, blocks_written): if self.mapped_cnt: assert blocks_written <= self.mapped_cnt percent = int((float(blocks_written) / self.mapped_cnt) * 100) - self._log.debug("wrote %d blocks out of %d (%d%%)" % - (blocks_written, self.mapped_cnt, percent)) + self._log.debug( + "wrote %d blocks out of %d (%d%%)" + % (blocks_written, self.mapped_cnt, percent) + ) else: self._log.debug("wrote %d blocks" % blocks_written) @@ -362,7 +379,7 @@ def _update_progress(self, blocks_written): return if self.mapped_cnt: - progress = '\r' + self._progress_format % percent + '\n' + progress = "\r" + self._progress_format % percent + "\n" else: # Do not rotate the wheel too fast now = datetime.datetime.now() @@ -371,8 +388,8 @@ def _update_progress(self, blocks_written): return self._progress_time = now - progress_wheel = ('-', '\\', '|', '/') - progress = '\r' + progress_wheel[self._progress_index % 4] + '\n' + progress_wheel = ("-", "\\", "|", "/") + progress = "\r" + progress_wheel[self._progress_index % 4] + "\n" self._progress_index += 1 # This is a little trick we do in order to make sure that the next @@ -382,7 +399,7 @@ def _update_progress(self, blocks_written): # exception - the error message will start form new line. if self._progress_started: # The "move cursor up" escape sequence - self._progress_file.write('\033[1A') # pylint: disable=W1401 + self._progress_file.write("\033[1A") # pylint: disable=W1401 else: self._progress_started = True @@ -426,7 +443,7 @@ def _get_block_ranges(self): # The range of blocks has the "X - Y" format, or it can be just "X" # in old bmap format versions. First, split the blocks range string # and strip white-spaces. - split = [x.strip() for x in blocks_range.split('-', 1)] + split = [x.strip() for x in blocks_range.split("-", 1)] first = int(split[0]) if len(split) > 1: @@ -488,9 +505,10 @@ def _get_data(self, verify): try: buf = self._f_image.read(length * self.block_size) except IOError as err: - raise Error("error while reading blocks %d-%d of the " - "image file '%s': %s" - % (start, end, self._image_path, err)) + raise Error( + "error while reading blocks %d-%d of the " + "image file '%s': %s" % (start, end, self._image_path, err) + ) if not buf: self._batch_queue.put(None) @@ -500,17 +518,19 @@ def _get_data(self, verify): hash_obj.update(buf) blocks = (len(buf) + self.block_size - 1) / self.block_size - self._log.debug("queueing %d blocks, queue length is %d" % - (blocks, self._batch_queue.qsize())) + self._log.debug( + "queueing %d blocks, queue length is %d" + % (blocks, self._batch_queue.qsize()) + ) - self._batch_queue.put(("range", start, start + blocks - 1, - buf)) + self._batch_queue.put(("range", start, start + blocks - 1, buf)) if verify and chksum and hash_obj.hexdigest() != chksum: - raise Error("checksum mismatch for blocks range %d-%d: " - "calculated %s, should be %s (image file %s)" - % (first, last, hash_obj.hexdigest(), - chksum, self._image_path)) + raise Error( + "checksum mismatch for blocks range %d-%d: " + "calculated %s, should be %s (image file %s)" + % (first, last, hash_obj.hexdigest(), chksum, self._image_path) + ) # Silence pylint warning about catching too general exception # pylint: disable=W0703 except Exception: @@ -532,7 +552,7 @@ def copy(self, sync=True, verify=True): # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. self._batch_queue = Queue.Queue(self._batch_queue_len) - thread.start_new_thread(self._get_data, (verify, )) + thread.start_new_thread(self._get_data, (verify,)) blocks_written = 0 bytes_written = 0 @@ -571,11 +591,13 @@ def copy(self, sync=True, verify=True): try: self._f_dest.write(buf) except IOError as err: - raise Error("error while writing blocks %d-%d of '%s': %s" - % (start, end, self._dest_path, err)) + raise Error( + "error while writing blocks %d-%d of '%s': %s" + % (start, end, self._dest_path, err) + ) self._batch_queue.task_done() - blocks_written += (end - start + 1) + blocks_written += end - start + 1 bytes_written += len(buf) self._update_progress(blocks_written) @@ -587,19 +609,25 @@ def copy(self, sync=True, verify=True): # This is just a sanity check - we should have written exactly # 'mapped_cnt' blocks. if blocks_written != self.mapped_cnt: - raise Error("wrote %u blocks from image '%s' to '%s', but should " - "have %u - bmap file '%s' does not belong to this " - "image" - % (blocks_written, self._image_path, self._dest_path, - self.mapped_cnt, self._bmap_path)) + raise Error( + "wrote %u blocks from image '%s' to '%s', but should " + "have %u - bmap file '%s' does not belong to this " + "image" + % ( + blocks_written, + self._image_path, + self._dest_path, + self.mapped_cnt, + self._bmap_path, + ) + ) if self._dest_is_regfile: # Make sure the destination file has the same size as the image try: os.ftruncate(self._f_dest.fileno(), self.image_size) except OSError as err: - raise Error("cannot truncate file '%s': %s" - % (self._dest_path, err)) + raise Error("cannot truncate file '%s': %s" % (self._dest_path, err)) try: self._f_dest.flush() @@ -619,8 +647,9 @@ def sync(self): try: os.fsync(self._f_dest.fileno()), except OSError as err: - raise Error("cannot synchronize '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot synchronize '%s': %s " % (self._dest_path, err.strerror) + ) class BmapBdevCopy(BmapCopy): @@ -654,19 +683,29 @@ def __init__(self, image, dest, bmap=None, image_size=None, log=None): bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: - raise Error("cannot seed block device '%s': %s " - % (self._dest_path, err.strerror)) + raise Error( + "cannot seed block device '%s': %s " + % (self._dest_path, err.strerror) + ) if bdev_size < self.image_size: - raise Error("the image file '%s' has size %s and it will not " - "fit the block device '%s' which has %s capacity" - % (self._image_path, self.image_size_human, - self._dest_path, human_size(bdev_size))) + raise Error( + "the image file '%s' has size %s and it will not " + "fit the block device '%s' which has %s capacity" + % ( + self._image_path, + self.image_size_human, + self._dest_path, + human_size(bdev_size), + ) + ) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev - self._sysfs_base = "/sys/dev/block/%s:%s/" % \ - (os.major(st_rdev), os.minor(st_rdev)) + self._sysfs_base = "/sys/dev/block/%s:%s/" % ( + os.major(st_rdev), + os.minor(st_rdev), + ) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we @@ -695,16 +734,18 @@ def _tune_block_device(self): f_scheduler.seek(0) f_scheduler.write("noop") except IOError as err: - self._log.warning("failed to enable I/O optimization, expect " - "suboptimal speed (reason: cannot switch " - "to the 'noop' I/O scheduler: %s)" % err) + self._log.warning( + "failed to enable I/O optimization, expect " + "suboptimal speed (reason: cannot switch " + "to the 'noop' I/O scheduler: %s)" % err + ) else: # The file contains a list of schedulers with the current # scheduler in square brackets, e.g., "noop deadline [cfq]". # Fetch the name of the current scheduler. import re - match = re.match(r'.*\[(.+)\].*', contents) + match = re.match(r".*\[(.+)\].*", contents) if match: self._old_scheduler_value = match.group(1) @@ -717,9 +758,11 @@ def _tune_block_device(self): f_ratio.seek(0) f_ratio.write("1") except IOError as err: - self._log.warning("failed to disable excessive buffering, expect " - "worse system responsiveness (reason: cannot set " - "max. I/O ratio to 1: %s)" % err) + self._log.warning( + "failed to disable excessive buffering, expect " + "worse system responsiveness (reason: cannot set " + "max. I/O ratio to 1: %s)" % err + ) def _restore_bdev_settings(self): """ @@ -732,16 +775,20 @@ def _restore_bdev_settings(self): with open(self._sysfs_scheduler_path, "w") as f_scheduler: f_scheduler.write(self._old_scheduler_value) except IOError as err: - raise Error("cannot restore the '%s' I/O scheduler: %s" - % (self._old_scheduler_value, err)) + raise Error( + "cannot restore the '%s' I/O scheduler: %s" + % (self._old_scheduler_value, err) + ) if self._old_max_ratio_value is not None: try: with open(self._sysfs_max_ratio_path, "w") as f_ratio: f_ratio.write(self._old_max_ratio_value) except IOError as err: - raise Error("cannot set the max. I/O ratio back to '%s': %s" - % (self._old_max_ratio_value, err)) + raise Error( + "cannot set the max. I/O ratio back to '%s': %s" + % (self._old_max_ratio_value, err) + ) def copy(self, sync=True, verify=True): """ diff --git a/tests/test_CLI.py b/tests/test_CLI.py index dfcd7a9..2330c70 100644 --- a/tests/test_CLI.py +++ b/tests/test_CLI.py @@ -22,41 +22,79 @@ import tempfile import tests.helpers + class TestCLI(unittest.TestCase): def test_valid_signature(self): - completed_process = subprocess.run ([ './bmaptool', 'copy', '--bmap', - 'tests/test-data/test.image.bmap.v2.0', '--bmap-sig', - 'tests/test-data/test.image.bmap.v2.0.valid-sig', - 'tests/test-data/test.image.gz', self.tmpfile ], check=False) + completed_process = subprocess.run( + [ + "./bmaptool", + "copy", + "--bmap", + "tests/test-data/test.image.bmap.v2.0", + "--bmap-sig", + "tests/test-data/test.image.bmap.v2.0.valid-sig", + "tests/test-data/test.image.gz", + self.tmpfile, + ], + check=False, + ) self.assertEqual(completed_process.returncode, 0) def test_unknown_signer(self): - completed_process = subprocess.run ([ './bmaptool', 'copy', '--bmap', - 'tests/test-data/test.image.bmap.v2.0', '--bmap-sig', - 'tests/test-data/test.image.bmap.v2.0.sig-by-wrong-key', - 'tests/test-data/test.image.gz', self.tmpfile ], check=False) + completed_process = subprocess.run( + [ + "./bmaptool", + "copy", + "--bmap", + "tests/test-data/test.image.bmap.v2.0", + "--bmap-sig", + "tests/test-data/test.image.bmap.v2.0.sig-by-wrong-key", + "tests/test-data/test.image.gz", + self.tmpfile, + ], + check=False, + ) self.assertEqual(completed_process.returncode, 1) - + def test_wrong_signature(self): - completed_process = subprocess.run ([ './bmaptool', 'copy', '--bmap', - 'tests/test-data/test.image.bmap.v1.4', '--bmap-sig', - 'tests/test-data/test.image.bmap.v2.0.valid-sig', - 'tests/test-data/test.image.gz', self.tmpfile ], check=False) + completed_process = subprocess.run( + [ + "./bmaptool", + "copy", + "--bmap", + "tests/test-data/test.image.bmap.v1.4", + "--bmap-sig", + "tests/test-data/test.image.bmap.v2.0.valid-sig", + "tests/test-data/test.image.gz", + self.tmpfile, + ], + check=False, + ) self.assertEqual(completed_process.returncode, 1) def test_wrong_signature_uknown_signer(self): - completed_process = subprocess.run ([ './bmaptool', 'copy', '--bmap', - 'tests/test-data/test.image.bmap.v1.4', '--bmap-sig', - 'tests/test-data/test.image.bmap.v2.0.sig-by-wrong-key', - 'tests/test-data/test.image.gz', self.tmpfile ], check=False) + completed_process = subprocess.run( + [ + "./bmaptool", + "copy", + "--bmap", + "tests/test-data/test.image.bmap.v1.4", + "--bmap-sig", + "tests/test-data/test.image.bmap.v2.0.sig-by-wrong-key", + "tests/test-data/test.image.gz", + self.tmpfile, + ], + check=False, + ) self.assertEqual(completed_process.returncode, 1) def setUp(self): - os.environ['GNUPGHOME']='tests/test-data/gnupg/' + os.environ["GNUPGHOME"] = "tests/test-data/gnupg/" self.tmpfile = tempfile.mkstemp(prefix="testfile_", dir=".")[1] def tearDown(self): os.unlink(self.tmpfile) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_api_base.py b/tests/test_api_base.py index 39b83fa..20e9694 100644 --- a/tests/test_api_base.py +++ b/tests/test_api_base.py @@ -47,6 +47,7 @@ class Error(Exception): """A class for exceptions generated by this test.""" + pass @@ -65,8 +66,10 @@ def _compare_holes(file1, file2): iterator = zip_longest(iterator1, iterator2) for range1, range2 in iterator: if range1 != range2: - raise Error("mismatch for hole %d-%d, it is %d-%d in file2" - % (range1[0], range1[1], range2[0], range2[1])) + raise Error( + "mismatch for hole %d-%d, it is %d-%d in file2" + % (range1[0], range1[1], range2[0], range2[1]) + ) def _generate_compressed_files(file_path, delete=True): @@ -80,27 +83,29 @@ def _generate_compressed_files(file_path, delete=True): # Make sure the temporary files start with the same name as 'file_obj' in # order to simplify debugging. - prefix = os.path.splitext(os.path.basename(file_path))[0] + '.' + prefix = os.path.splitext(os.path.basename(file_path))[0] + "." # Put the temporary files in the directory with 'file_obj' directory = os.path.dirname(file_path) - compressors = [("bzip2", None, ".bz2", "-c -k"), - ("pbzip2", None, ".p.bz2", "-c -k"), - ("gzip", None, ".gz", "-c"), - ("pigz", None, ".p.gz", "-c -k"), - ("xz", None, ".xz", "-c -k"), - ("lzop", None, ".lzo", "-c -k"), - ("lz4", None, ".lz4", "-c -k"), - ("zstd", None, ".zst", "-c -k"), - # The "-P -C /" trick is used to avoid silly warnings: - # "tar: Removing leading `/' from member names" - ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"), - ("gzip", "tar", ".tar.gz", "-c -z -O -P -C /"), - ("xz", "tar", ".tar.xz", "-c -J -O -P -C /"), - ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C /"), - ("lz4", "tar", ".tar.lz4", "-c -Ilz4 -O -P -C /"), - ("zstd", "tar", ".tar.zst", "-c -Izstd -O -P -C /"), - ("zip", None, ".zip", "-q -j -")] + compressors = [ + ("bzip2", None, ".bz2", "-c -k"), + ("pbzip2", None, ".p.bz2", "-c -k"), + ("gzip", None, ".gz", "-c"), + ("pigz", None, ".p.gz", "-c -k"), + ("xz", None, ".xz", "-c -k"), + ("lzop", None, ".lzo", "-c -k"), + ("lz4", None, ".lz4", "-c -k"), + ("zstd", None, ".zst", "-c -k"), + # The "-P -C /" trick is used to avoid silly warnings: + # "tar: Removing leading `/' from member names" + ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"), + ("gzip", "tar", ".tar.gz", "-c -z -O -P -C /"), + ("xz", "tar", ".tar.xz", "-c -J -O -P -C /"), + ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C /"), + ("lz4", "tar", ".tar.lz4", "-c -Ilz4 -O -P -C /"), + ("zstd", "tar", ".tar.zst", "-c -Izstd -O -P -C /"), + ("zip", None, ".zip", "-q -j -"), + ] for decompressor, archiver, suffix, options in compressors: if not BmapHelpers.program_is_available(decompressor): @@ -108,9 +113,9 @@ def _generate_compressed_files(file_path, delete=True): if archiver and not BmapHelpers.program_is_available(archiver): continue - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix=suffix) + tmp_file_obj = tempfile.NamedTemporaryFile( + "wb+", prefix=prefix, delete=delete, dir=directory, suffix=suffix + ) if archiver: args = archiver + " " + options + " " + file_path @@ -138,27 +143,27 @@ def _do_test(image, image_size, delete=True): try: Filemap.filemap(image) except Filemap.ErrorNotSupp as e: - sys.stderr.write('%s\n' % e) + sys.stderr.write("%s\n" % e) return # Make sure the temporary files start with the same name as 'image' in # order to simplify debugging. - prefix = os.path.splitext(os.path.basename(image))[0] + '.' + prefix = os.path.splitext(os.path.basename(image))[0] + "." # Put the temporary files in the directory with the image directory = os.path.dirname(image) # Create and open a temporary file for a copy of the image - f_copy = tempfile.NamedTemporaryFile("wb+", prefix=prefix, - delete=delete, dir=directory, - suffix=".copy") + f_copy = tempfile.NamedTemporaryFile( + "wb+", prefix=prefix, delete=delete, dir=directory, suffix=".copy" + ) # Create and open 2 temporary files for the bmap - f_bmap1 = tempfile.NamedTemporaryFile("w+", prefix=prefix, - delete=delete, dir=directory, - suffix=".bmap1") - f_bmap2 = tempfile.NamedTemporaryFile("w+", prefix=prefix, - delete=delete, dir=directory, - suffix=".bmap2") + f_bmap1 = tempfile.NamedTemporaryFile( + "w+", prefix=prefix, delete=delete, dir=directory, suffix=".bmap1" + ) + f_bmap2 = tempfile.NamedTemporaryFile( + "w+", prefix=prefix, delete=delete, dir=directory, suffix=".bmap2" + ) image_chksum = helpers.calculate_chksum(image) @@ -170,8 +175,9 @@ def _do_test(image, image_size, delete=True): creator = BmapCreate.BmapCreate(image, f_bmap1.name) creator.generate() - helpers.copy_and_verify_image(image, f_copy.name, f_bmap1.name, - image_chksum, image_size) + helpers.copy_and_verify_image( + image, f_copy.name, f_bmap1.name, image_chksum, image_size + ) # Make sure that holes in the copy are identical to holes in the random # sparse file. @@ -183,8 +189,9 @@ def _do_test(image, image_size, delete=True): creator = BmapCreate.BmapCreate(image, f_bmap2) creator.generate() - helpers.copy_and_verify_image(image, f_copy.name, f_bmap2.name, - image_chksum, image_size) + helpers.copy_and_verify_image( + image, f_copy.name, f_bmap2.name, image_chksum, image_size + ) _compare_holes(image, f_copy.name) # Make sure the bmap files generated at pass 1 and pass 2 are identical @@ -195,26 +202,29 @@ def _do_test(image, image_size, delete=True): # for compressed in _generate_compressed_files(image, delete=delete): - helpers.copy_and_verify_image(compressed, f_copy.name, - f_bmap1.name, image_chksum, image_size) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, image_size + ) # Test without setting the size - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, None) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, None + ) # Append a "file:" prefixe to make BmapCopy use urllib compressed = "file:" + compressed - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, image_size) - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, None) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, image_size + ) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, None + ) # # Pass 5: copy without bmap and make sure it is identical to the original # file. - helpers.copy_and_verify_image(image, f_copy.name, None, image_chksum, - image_size) + helpers.copy_and_verify_image(image, f_copy.name, None, image_chksum, image_size) helpers.copy_and_verify_image(image, f_copy.name, None, image_chksum, None) # @@ -222,18 +232,22 @@ def _do_test(image, image_size, delete=True): # for compressed in _generate_compressed_files(image, delete=delete): - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, image_size) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, image_size + ) # Test without setting the size - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, None) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, None + ) # Append a "file:" prefix to make BmapCopy use urllib - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, image_size) - helpers.copy_and_verify_image(compressed, f_copy.name, f_bmap1.name, - image_chksum, None) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, image_size + ) + helpers.copy_and_verify_image( + compressed, f_copy.name, f_bmap1.name, image_chksum, None + ) # Close temporary files, which will also remove them f_copy.close() @@ -256,10 +270,9 @@ def test(self): # pylint: disable=R0201 # Delete all the test-related temporary files automatically delete = True # Create all the test-related temporary files in current directory - directory = '.' + directory = "." - iterator = helpers.generate_test_files(delete=delete, - directory=directory) + iterator = helpers.generate_test_files(delete=delete, directory=directory) for f_image, image_size, _, _ in iterator: assert image_size == os.path.getsize(f_image.name) _do_test(f_image.name, image_size, delete=delete) diff --git a/tests/test_bmap_helpers.py b/tests/test_bmap_helpers.py index 36c4557..233aece 100644 --- a/tests/test_bmap_helpers.py +++ b/tests/test_bmap_helpers.py @@ -21,13 +21,14 @@ import os import sys import tempfile + try: from unittest.mock import patch -except ImportError: # for Python < 3.3 +except ImportError: # for Python < 3.3 from mock import patch try: from tempfile import TemporaryDirectory -except ImportError: # for Python < 3.2 +except ImportError: # for Python < 3.2 from backports.tempfile import TemporaryDirectory from bmaptools import BmapHelpers @@ -45,8 +46,9 @@ class TestBmapHelpers(unittest.TestCase): def test_get_file_system_type(self): """Check a file system type is returned when used with a file""" - with tempfile.NamedTemporaryFile("r", prefix="testfile_", - delete=True, dir=".", suffix=".img") as fobj: + with tempfile.NamedTemporaryFile( + "r", prefix="testfile_", delete=True, dir=".", suffix=".img" + ) as fobj: fstype = BmapHelpers.get_file_system_type(fobj.name) self.assertTrue(fstype) @@ -62,8 +64,9 @@ def test_get_file_system_type_symlink(self): """Check a file system type is returned when used with a symlink""" with TemporaryDirectory(prefix="testdir_", dir=".") as directory: - fobj = tempfile.NamedTemporaryFile("r", prefix="testfile_", delete=False, - dir=directory, suffix=".img") + fobj = tempfile.NamedTemporaryFile( + "r", prefix="testfile_", delete=False, dir=directory, suffix=".img" + ) lnk = os.path.join(directory, "test_symlink") os.symlink(fobj.name, lnk) fstype = BmapHelpers.get_file_system_type(lnk) @@ -72,20 +75,21 @@ def test_get_file_system_type_symlink(self): def test_is_zfs_configuration_compatible_enabled(self): """Check compatiblilty check is true when zfs param is set correctly""" - with tempfile.NamedTemporaryFile("w+", prefix="testfile_", - delete=True, dir=".", suffix=".txt") as fobj: + with tempfile.NamedTemporaryFile( + "w+", prefix="testfile_", delete=True, dir=".", suffix=".txt" + ) as fobj: fobj.write("1") fobj.flush() mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) with mockobj: self.assertTrue(BmapHelpers.is_zfs_configuration_compatible()) - def test_is_zfs_configuration_compatible_disabled(self): """Check compatiblilty check is false when zfs param is set incorrectly""" - with tempfile.NamedTemporaryFile("w+", prefix="testfile_", - delete=True, dir=".", suffix=".txt") as fobj: + with tempfile.NamedTemporaryFile( + "w+", prefix="testfile_", delete=True, dir=".", suffix=".txt" + ) as fobj: fobj.write("0") fobj.flush() mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) @@ -95,8 +99,9 @@ def test_is_zfs_configuration_compatible_disabled(self): def test_is_zfs_configuration_compatible_invalid_read_value(self): """Check error raised if any content of zfs config file invalid""" - with tempfile.NamedTemporaryFile("a", prefix="testfile_", - delete=True, dir=".", suffix=".txt") as fobj: + with tempfile.NamedTemporaryFile( + "a", prefix="testfile_", delete=True, dir=".", suffix=".txt" + ) as fobj: mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) with self.assertRaises(BmapHelpers.Error): with mockobj: @@ -121,11 +126,14 @@ def test_is_zfs_configuration_compatible_notinstalled(self): self.assertFalse(BmapHelpers.is_zfs_configuration_compatible()) @patch.object(BmapHelpers, "get_file_system_type", return_value="zfs") - def test_is_compatible_file_system_zfs_valid(self, mock_get_fs_type): #pylint: disable=unused-argument + def test_is_compatible_file_system_zfs_valid( + self, mock_get_fs_type + ): # pylint: disable=unused-argument """Check compatiblilty check passes when zfs param is set correctly""" - with tempfile.NamedTemporaryFile("w+", prefix="testfile_", - delete=True, dir=".", suffix=".img") as fobj: + with tempfile.NamedTemporaryFile( + "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" + ) as fobj: fobj.write("1") fobj.flush() mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) @@ -133,11 +141,14 @@ def test_is_compatible_file_system_zfs_valid(self, mock_get_fs_type): #pylint: d self.assertTrue(BmapHelpers.is_compatible_file_system(fobj.name)) @patch.object(BmapHelpers, "get_file_system_type", return_value="zfs") - def test_is_compatible_file_system_zfs_invalid(self, mock_get_fs_type): #pylint: disable=unused-argument + def test_is_compatible_file_system_zfs_invalid( + self, mock_get_fs_type + ): # pylint: disable=unused-argument """Check compatiblilty check fails when zfs param is set incorrectly""" - with tempfile.NamedTemporaryFile("w+", prefix="testfile_", - delete=True, dir=".", suffix=".img") as fobj: + with tempfile.NamedTemporaryFile( + "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" + ) as fobj: fobj.write("0") fobj.flush() mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) @@ -145,9 +156,12 @@ def test_is_compatible_file_system_zfs_invalid(self, mock_get_fs_type): #pylint: self.assertFalse(BmapHelpers.is_compatible_file_system(fobj.name)) @patch.object(BmapHelpers, "get_file_system_type", return_value="ext4") - def test_is_compatible_file_system_ext4(self, mock_get_fs_type): #pylint: disable=unused-argument + def test_is_compatible_file_system_ext4( + self, mock_get_fs_type + ): # pylint: disable=unused-argument """Check non-zfs file systems pass compatiblilty checks""" - with tempfile.NamedTemporaryFile("w+", prefix="testfile_", - delete=True, dir=".", suffix=".img") as fobj: + with tempfile.NamedTemporaryFile( + "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" + ) as fobj: self.assertTrue(BmapHelpers.is_compatible_file_system(fobj.name)) diff --git a/tests/test_compat.py b/tests/test_compat.py index a875929..61fa365 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -55,8 +55,7 @@ class TestCompat(unittest.TestCase): def test(self): """The test entry point.""" - test_data_dir = os.path.join(os.path.dirname(__file__), - _TEST_DATA_SUBDIR) + test_data_dir = os.path.join(os.path.dirname(__file__), _TEST_DATA_SUBDIR) image_path = os.path.join(test_data_dir, _IMAGE_NAME) # Construct the list of bmap files to test @@ -67,10 +66,12 @@ def test(self): self._bmap_paths.append(dentry_path) # Create and open a temporary file for uncompressed image and its copy - self._f_image = tempfile.NamedTemporaryFile("wb+", prefix=_IMAGE_NAME, - suffix=".image") - self._f_copy = tempfile.NamedTemporaryFile("wb+", prefix=_IMAGE_NAME, - suffix=".copy") + self._f_image = tempfile.NamedTemporaryFile( + "wb+", prefix=_IMAGE_NAME, suffix=".image" + ) + self._f_copy = tempfile.NamedTemporaryFile( + "wb+", prefix=_IMAGE_NAME, suffix=".copy" + ) # Uncompress the test image into 'self._f_image' f_tmp_img = TransRead.TransRead(image_path) @@ -83,9 +84,9 @@ def test(self): # Test the current version of BmapCopy for bmap_path in self._bmap_paths: - helpers.copy_and_verify_image(image_path, self._f_copy.name, - bmap_path, image_chksum, - image_size) + helpers.copy_and_verify_image( + image_path, self._f_copy.name, bmap_path, image_chksum, image_size + ) # Test the older versions of BmapCopy self._test_older_bmapcopy() @@ -104,8 +105,7 @@ def import_module(searched_module): modref = getattr(modref, name) return modref - oldcodebase_dir = os.path.join(os.path.dirname(__file__), - _OLDCODEBASE_SUBDIR) + oldcodebase_dir = os.path.join(os.path.dirname(__file__), _OLDCODEBASE_SUBDIR) # Construct the list of old BmapCopy modules old_modules = [] @@ -148,8 +148,10 @@ def _do_test_older_bmapcopy(self, bmap_path, modref): # was a screw-up, because it actually had incompatible changes, # so old versions of BmapCopy are supposed to fail. if not (supported_ver == 1 and bmap_version == "1.4"): - print("Module \"%s\" failed to handle \"%s\"" % - (modref.__name__, bmap_path)) + print( + 'Module "%s" failed to handle "%s"' + % (modref.__name__, bmap_path) + ) raise f_bmap.close() diff --git a/tests/test_filemap.py b/tests/test_filemap.py index 63ac312..61e7bf1 100644 --- a/tests/test_filemap.py +++ b/tests/test_filemap.py @@ -41,11 +41,11 @@ class Error(Exception): """A class for exceptions generated by this test.""" + pass -def _check_ranges(f_image, filemap, first_block, blocks_cnt, - ranges, ranges_type): +def _check_ranges(f_image, filemap, first_block, blocks_cnt, ranges, ranges_type): """ This is a helper function for '_do_test()' which compares the correct 'ranges' list of mapped or unmapped blocks ranges for file object 'f_image' @@ -69,8 +69,7 @@ def _check_ranges(f_image, filemap, first_block, blocks_cnt, # starting from block 'first_block'. Create an iterator which filters # those block ranges from the 'ranges' list, that are out of the # 'first_block'/'blocks_cnt' file region. - ranges_iterator = (x for x in ranges if x[1] >= first_block and - x[0] <= last_block) + ranges_iterator = (x for x in ranges if x[1] >= first_block and x[0] <= last_block) iterator = zip_longest(ranges_iterator, filemap_iterator) # Iterate over both - the (filtered) 'ranges' list which contains correct @@ -86,21 +85,35 @@ def _check_ranges(f_image, filemap, first_block, blocks_cnt, correct = (correct[0], last_block) if check[0] > check[1] or check != correct: - raise Error("bad or unmatching %s range for file '%s': correct " - "is %d-%d, get_%s_ranges(%d, %d) returned %d-%d" - % (ranges_type, f_image.name, correct[0], correct[1], - ranges_type, first_block, blocks_cnt, - check[0], check[1])) + raise Error( + "bad or unmatching %s range for file '%s': correct " + "is %d-%d, get_%s_ranges(%d, %d) returned %d-%d" + % ( + ranges_type, + f_image.name, + correct[0], + correct[1], + ranges_type, + first_block, + blocks_cnt, + check[0], + check[1], + ) + ) for block in range(correct[0], correct[1] + 1): if ranges_type == "mapped" and filemap.block_is_unmapped(block): - raise Error("range %d-%d of file '%s' is mapped, but" - "'block_is_unmapped(%d) returned 'True'" - % (correct[0], correct[1], f_image.name, block)) + raise Error( + "range %d-%d of file '%s' is mapped, but" + "'block_is_unmapped(%d) returned 'True'" + % (correct[0], correct[1], f_image.name, block) + ) if ranges_type == "unmapped" and filemap.block_is_mapped(block): - raise Error("range %d-%d of file '%s' is unmapped, but" - "'block_is_mapped(%d) returned 'True'" - % (correct[0], correct[1], f_image.name, block)) + raise Error( + "range %d-%d of file '%s' is unmapped, but" + "'block_is_mapped(%d) returned 'True'" + % (correct[0], correct[1], f_image.name, block) + ) def _do_test(f_image, filemap, mapped, unmapped): @@ -116,17 +129,14 @@ def _do_test(f_image, filemap, mapped, unmapped): first_block = 0 blocks_cnt = filemap.blocks_cnt _check_ranges(f_image, filemap, first_block, blocks_cnt, mapped, "mapped") - _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, - "unmapped") + _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, "unmapped") # Select a random area in the file and repeat the test few times for _ in range(0, 10): first_block = random.randint(0, filemap.blocks_cnt - 1) blocks_cnt = random.randint(1, filemap.blocks_cnt - first_block) - _check_ranges(f_image, filemap, first_block, blocks_cnt, mapped, - "mapped") - _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, - "unmapped") + _check_ranges(f_image, filemap, first_block, blocks_cnt, mapped, "mapped") + _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, "unmapped") class TestFilemap(unittest.TestCase): @@ -144,12 +154,11 @@ def test(self): # pylint: disable=R0201 # Delete all the test-related temporary files automatically delete = True # Create all the test-related temporary files in current directory - directory = '.' + directory = "." # Maximum size of the random files used in this test max_size = 16 * 1024 * 1024 - iterator = tests.helpers.generate_test_files(max_size, directory, - delete) + iterator = tests.helpers.generate_test_files(max_size, directory, delete) for f_image, _, mapped, unmapped in iterator: try: fiemap = Filemap.FilemapFiemap(f_image)