From 4cfcf34d90d1258cf4d90e612616287ec225c7a0 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 25 May 2016 18:18:15 +0200 Subject: [PATCH 01/55] Fix names in __all__, and super call (#59) Signed-off-by: Lukas Bednar --- rrmngmnt/__init__.py | 14 +++++++------- rrmngmnt/user.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rrmngmnt/__init__.py b/rrmngmnt/__init__.py index a952651..1cfbccc 100644 --- a/rrmngmnt/__init__.py +++ b/rrmngmnt/__init__.py @@ -10,11 +10,11 @@ __all__ = [ - Host, - User, - RootUser, - Domain, - InternalDomain, - ADUser, - Database, + 'Host', + 'User', + 'RootUser', + 'Domain', + 'InternalDomain', + 'ADUser', + 'Database', ] diff --git a/rrmngmnt/user.py b/rrmngmnt/user.py index 2fd0b94..c9737c1 100644 --- a/rrmngmnt/user.py +++ b/rrmngmnt/user.py @@ -9,7 +9,7 @@ def __init__(self, name, password): :param password: password :type password: str """ - super(Resource, self).__init__() + super(User, self).__init__() self.name = name self.password = password From e6c70e6e0efd75e1fd350e9ac1ce375026e63f44 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Thu, 26 May 2016 11:40:43 +0200 Subject: [PATCH 02/55] Enable code coverage monitoring (#60) --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 39a6a06..926b3ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - "2.7" - "3.4" install: - - pip install tox + - pip install tox codecov script: - tox +after_success: + - codecov From da13aace9dafaebf6a09e5706d8e361aa96b3242 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Thu, 26 May 2016 12:31:39 +0200 Subject: [PATCH 03/55] Add code coverage and health badge to readme (#61) --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1431093..03edd07 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ +[![Build Status][travisimg]][travis] +[![Code Coverage][codecovimg]][codecov] +[![Code Health][codehealthimg]][codehealth] + # python-rrmngmnt Remote Resources MaNaGeMeNT -[![Build Status](https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master)](https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt) ## Intro This tool helps you manage remote machines and services running on that. @@ -131,3 +134,10 @@ h.power_manager.restart() ```python python setup.py devop ``` + +[travisimg]: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master +[travis]: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt +[codecovimg]: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt/branch/master/graph/badge.svg +[codecov]: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt +[codehealthimg]: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master/landscape.svg?style=flat +[codehealth]: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master From ede0c9640bd9ab51c469e42e9afcf35b24554595 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Tue, 7 Jun 2016 18:56:37 +0300 Subject: [PATCH 04/55] Fix when FQDN is not resolved by socket.getfqdn(ip) (#62) * Set FQDN if host created qith FQDN Sometime socket.getfqdn(ip) return IP and not the host FQDN so if Host was created with FQDN we will use it as FQDN * Set FQDN if host created qith FQDN Sometime socket.getfqdn(ip) return IP and not the host FQDN so if Host was created with FQDN we will use it as FQDN * Set FQDN if host created qith FQDN Sometime socket.getfqdn(ip) return IP and not the host FQDN so if Host was created with FQDN we will use it as FQDN * Set FQDN if host created qith FQDN Sometime socket.getfqdn(ip) return IP and not the host FQDN so if Host was created with FQDN we will use it as FQDN --- rrmngmnt/host.py | 4 +++- tests/test_host.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 64d6e77..ac7feb0 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -59,7 +59,9 @@ def __init__(self, ip, service_provider=None): :type service_provider: class which implement SystemService interface """ super(Host, self).__init__() + self._fqdn = None if not netaddr.valid_ipv4(ip): + self._fqdn = ip ip = fqdn2ip(ip) self.ip = ip self.users = list() @@ -102,7 +104,7 @@ def add(self): @property def fqdn(self): - return socket.getfqdn(self.ip) + return socket.getfqdn(self.ip) if not self._fqdn else self._fqdn def add_power_manager(self, pm_type, **init_params): """ diff --git a/tests/test_host.py b/tests/test_host.py index 9e0116f..2d8975e 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -30,3 +30,16 @@ def test_executor_user(self): h.executor_user = user e = h.executor() e.user.name == 'lukas' + + +class TestHostFqdnIp(object): + + def test_host_ip(self): + h = Host('127.0.0.1') + assert h.ip == '127.0.0.1' + assert h.fqdn == 'localhost' + + def test_host_fqdn(self): + h = Host('localhost') + assert h.ip == '127.0.0.1' + assert h.fqdn == 'localhost' From 4326f0e62085f7ca6007f5e0ad6619b39ffba623 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Mon, 20 Jun 2016 14:05:53 +0300 Subject: [PATCH 05/55] Copy file with correct permissions and owner (#58) * Copy file with correct permissions and owner * Patch Set #2 - move function to operatingsystem module to be consistent with os.stat - add function get_file_stats - add positive and negative tests * Patch Set #3 - fix tests * fix * Patch Set #4 - change function names - get octal permissions from stat * Patch Set #5 - fix tests - decode all output to UTF-8 * Patch Set #6 - remove decode of output * Patch Set #7 - fix functions names under copy_to function * Patch Set #8 - give possbility to change permission and ownership of the file, when file copied from other resource I believe it no need to check if ownership is a tuple, because we explicitly define it under docstring * Patch Set #9 - fix call to function fs.chown --- rrmngmnt/filesystem.py | 6 +- rrmngmnt/host.py | 27 ++++--- rrmngmnt/operatingsystem.py | 124 +++++++++++++++++++++++++---- tests/test_os.py | 151 ++++++++++++++++++++++++++++++++++++ 4 files changed, 284 insertions(+), 24 deletions(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 278887e..310dd20 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -1,6 +1,7 @@ import os -from rrmngmnt.service import Service + from rrmngmnt import errors +from rrmngmnt.service import Service class FileSystem(Service): @@ -10,11 +11,12 @@ class FileSystem(Service): """ def _exec_command(self, cmd): host_executor = self.host.executor() - rc, _, err = host_executor.run_cmd(cmd) + rc, out, err = host_executor.run_cmd(cmd) if rc: raise errors.CommandExecutionFailure( cmd=cmd, executor=host_executor, rc=rc, err=err ) + return out def _exec_file_test(self, op, path): return self.host.executor().run_cmd( diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index ac7feb0..c453e1d 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -3,23 +3,24 @@ It should hold methods / properties which returns you Instance of specific Service hosted on that Host. """ -import os import copy +import os import socket -import netaddr import warnings -from rrmngmnt import ssh +import netaddr + from rrmngmnt import errors from rrmngmnt import power_manager +from rrmngmnt import ssh from rrmngmnt.common import fqdn2ip -from rrmngmnt.network import Network -from rrmngmnt.storage import NFSService, LVMService -from rrmngmnt.service import Systemd, SysVinit, InitCtl -from rrmngmnt.resource import Resource from rrmngmnt.filesystem import FileSystem -from rrmngmnt.package_manager import PackageManagerProxy +from rrmngmnt.network import Network from rrmngmnt.operatingsystem import OperatingSystem +from rrmngmnt.package_manager import PackageManagerProxy +from rrmngmnt.resource import Resource +from rrmngmnt.service import Systemd, SysVinit, InitCtl +from rrmngmnt.storage import NFSService, LVMService class Host(Resource): @@ -243,7 +244,7 @@ def run_command( ) return rc, out, err - def copy_to(self, resource, src, dst): + def copy_to(self, resource, src, dst, mode=None, ownership=None): """ Copy to host from another resource @@ -253,12 +254,20 @@ def copy_to(self, resource, src, dst): :type src: str :param dst: path to destination :type dst: str + :param mode: file permissions + :type mode: str + :param ownership: file ownership(ex. ('root', 'root')) + :type ownership: tuple """ with resource.executor().session() as resource_session: with self.executor().session() as host_session: with resource_session.open_file(src, 'rb') as resource_file: with host_session.open_file(dst, 'wb') as host_file: host_file.write(resource_file.read()) + if mode: + self.fs.chmod(path=dst, mode=mode) + if ownership: + self.fs.chown(dst, *ownership) def _create_service(self, name, timeout): for provider in self.default_service_providers: diff --git a/rrmngmnt/operatingsystem.py b/rrmngmnt/operatingsystem.py index 517e4b2..a145410 100644 --- a/rrmngmnt/operatingsystem.py +++ b/rrmngmnt/operatingsystem.py @@ -14,15 +14,22 @@ def __init__(self, host): self._release_info = None self._dist = None - def get_release_str(self): - cmd = ['cat', '/etc/system-release'] - executor = self.host.executor() - rc, out, err = executor.run_cmd(cmd) + def _exec_command(self, cmd, err_msg=None): + host_executor = self.host.executor() + rc, out, err = host_executor.run_cmd(cmd) + if err_msg: + err = "{err_msg}: {err}".format(err_msg=err_msg, err=err) if rc: raise errors.CommandExecutionFailure( - executor, cmd, rc, - "Failed to obtain release string: {0}".format(err) + executor=host_executor, cmd=cmd, rc=rc, err=err ) + return out + + def get_release_str(self): + cmd = ['cat', '/etc/system-release'] + out = self._exec_command( + cmd=cmd, err_msg="Failed to obtain release string" + ) return out.strip() @property @@ -92,13 +99,9 @@ def get_distribution(self): "python", "-c", "import platform;print(','.join(platform.linux_distribution()))" ] - executor = self.host.executor() - rc, out, err = executor.run_cmd(cmd) - if rc: - raise errors.CommandExecutionFailure( - executor, cmd, rc, - "Failed to obtain release info: {0}".format(err) - ) + out = self._exec_command( + cmd=cmd, err_msg="Failed to obtain release info" + ) Distribution = namedtuple('Distribution', values) return Distribution(*[i.strip() for i in out.split(",")]) @@ -107,3 +110,98 @@ def distribution(self): if not self._dist: self._dist = self.get_distribution() return self._dist + + def stat(self, path): + """ + Get file or directory stats + + :return: file stats + :rtype: collections.namedtuple + """ + type_map = { + 'st_mode': ('0x%f', lambda x: int(x, 16)), + 'st_ino': ('%i', int), + 'st_dev': ('%d', int), + 'st_nlink': ('%h', int), + 'st_uid': ('%u', int), + 'st_gid': ('%g', int), + 'st_size': ('%s', int), + 'st_atime': ('%X', int), + 'st_mtime': ('%Y', int), + 'st_ctime': ('%W', int), + 'st_blocks': ('%b', int), + 'st_blksize': ('%o', int), + 'st_rdev': ('%t', int), + } + posix_stat_result = namedtuple( + "posix_stat_result", type_map.keys() + ) + + cmd = [ + "stat", + "-c", + ",".join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]), + path + ] + out = self._exec_command(cmd=cmd) + out = out.strip().split(',') + + data = {} + + for pair in out: + key, value = pair.split('=') + data[key] = type_map[key][1](value) + + return posix_stat_result(**data) + + def get_file_permissions(self, path): + """ + Get file permissions + + :return: file permission in octal form(example 0644) + :rtype: str + """ + cmd = ["stat", "-c", "%a", path] + return self._exec_command(cmd=cmd).strip() + + def get_file_owner(self, path): + """ + Get file user and group owner name + + :return: file user and group owner names(example ['root', 'root']) + :rtype: list + """ + cmd = ["stat", "-c", "%U %G", path] + return self._exec_command(cmd=cmd).split() + + def user_exists(self, user_name): + """ + Check if user exist on system + + :param user_name: user name + :type user_name: str + :return: True, if user exist, otherwise False + :rtype: bool + """ + try: + cmd = ["id", "-u", user_name] + self._exec_command(cmd=cmd) + except errors.CommandExecutionFailure: + return False + return True + + def group_exists(self, group_name): + """" + Check if group exist on system + + :param group_name: group name + :type group_name: str + :return: True, if group exist, otherwise False + :rtype: bool + """ + try: + cmd = ["id", "-g", group_name] + self._exec_command(cmd=cmd) + except errors.CommandExecutionFailure: + return False + return True diff --git a/tests/test_os.py b/tests/test_os.py index 9622e74..1377029 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -175,3 +175,154 @@ def test_get_release_info(self): info = self.get_host().os.release_info assert 'VERSION_ID' not in info assert len(info) == 4 + + +type_map = { + 'st_mode': ('0x%f', lambda x: int(x, 16)), + 'st_ino': ('%i', int), + 'st_dev': ('%d', int), + 'st_nlink': ('%h', int), + 'st_uid': ('%u', int), + 'st_gid': ('%g', int), + 'st_size': ('%s', int), + 'st_atime': ('%X', int), + 'st_mtime': ('%Y', int), + 'st_ctime': ('%W', int), + 'st_blocks': ('%b', int), + 'st_blksize': ('%o', int), + 'st_rdev': ('%t', int), +} + + +class TestFileStats(object): + data = { + 'stat -c %s /tmp/test' % + ','.join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]): ( + 0, + ( + 'st_ctime=0,' + 'st_rdev=0,' + 'st_blocks=1480,' + 'st_nlink=1,' + 'st_gid=0,' + 'st_dev=2051,' + 'st_ino=11804680,' + 'st_mode=0x81a4,' + 'st_mtime=1463487739,' + 'st_blksize=4096,' + 'st_size=751764,' + 'st_uid=0,' + 'st_atime=1463487196' + ), + '' + ), + 'stat -c "%U %G" /tmp/test': ( + 0, + 'root root', + '' + ), + 'stat -c %a /tmp/test': ( + 0, + '644\n', + '' + ), + 'id -u root': ( + 0, + '', + '' + ), + 'id -g root': ( + 0, + '', + '' + ) + } + files = {} + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + return Host(ip) + + def test_get_file_stats(self): + file_stats = self.get_host().os.stat('/tmp/test') + assert ( + file_stats.st_mode == 33188 and + file_stats.st_uid == 0 and + file_stats.st_gid == 0 + ) + + def test_get_file_owner(self): + file_user, file_group = self.get_host().os.get_file_owner('/tmp/test') + assert file_user == 'root' and file_group == 'root' + + def test_get_file_permissions(self): + assert self.get_host().os.get_file_permissions('/tmp/test') == '644' + + def test_user_exists(self): + assert self.get_host().os.user_exists('root') + + def test_group_exists(self): + assert self.get_host().os.group_exists('root') + + +class TestFileStatsNegative(object): + data = { + 'stat -c %s /tmp/negative_test' % + ','.join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]): ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'stat -c "%U %G" /tmp/negative_test': ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'stat -c %a /tmp/negative_test': ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'id -u test': ( + 1, + '', + '' + ), + 'id -g test': ( + 1, + '', + '' + ) + } + files = {} + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + return Host(ip) + + def test_get_file_stats(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.stat('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_get_file_owner(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.get_file_owner('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_get_file_permissions(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.get_file_permissions('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_user_exists(self): + assert not self.get_host().os.user_exists('test') + + def test_group_exists(self): + assert not self.get_host().os.group_exists('test') From b6ea84567a40baefa63f6e41f081fdfbb03164db Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 22 Jun 2016 16:32:26 +0200 Subject: [PATCH 06/55] Fixes issue #52 (#63) * move to rst format * move requirements under PM section * update requirements --- README.md | 143 ----------------------------------------- README.rst | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 184 insertions(+), 144 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index 03edd07..0000000 --- a/README.md +++ /dev/null @@ -1,143 +0,0 @@ -[![Build Status][travisimg]][travis] -[![Code Coverage][codecovimg]][codecov] -[![Code Health][codehealthimg]][codehealth] - -# python-rrmngmnt -Remote Resources MaNaGeMeNT - -## Intro -This tool helps you manage remote machines and services running on that. -It is targeted to Linux based machines. All is done via SSH connection, -that means SSH server must be running there already. -```python -from rrmngmnt import Host, RootUser - -h = Host("10.11.12.13") -h.users.append(RootUser('123456')) -exec = h.executor() -print exec.run_cmd(['echo', 'Hello World']) -``` - -## Features -List of provided interfaces to manage resources on machine, and examples. - -### Filesystem -Basic file operations, you can find there subset of python 'os' module related -to files. -```python -print h.fs.exists("/path/to/file") -h.fs.chown("/path/to/file", "root", "root") -h.fs.chmod("/path/to/file", "644") -h.fs.unlink("/path/to/file") -``` - -### Network -It allows to manage network configuration. -```python -print h.network.hostname -h.network.hostname = "my.machine.org" -print h.network.all_interfaces() -print h.network.list_bridges() -``` - -### Package Management -It encapsulates various package managements. It is able to determine -which package management to use. You can still specify package management -explicitly. - -Implemented managements: - - * APT - * YUM - * DNF - * RPM - -```python -# install htop package using implicit management -h.package_management.install('htop') -# remove htop package using rpm explicitly -h.package_management('rpm').remove('htop') -``` - -### System Services -You can toggle system services, it encapsulates various service managements. -It is able to determine which service management to use in most cases. - -Implemented managements: - - * Systemd - * SysVinit - * InitCtl - -```python -if h.service('httpd').status(): - h.service('httpd').stop() -if h.service('httpd').is_enabled(): - h.service('httpd').disable() -``` - -### Operating System Info -Host provide `os` attribute which allows obtain basic operating system info. -Note that `os.release_info` depends on systemd init system. - -```python -print h.os.distribution -# Distribution(distname='Fedora', version='23', id='Twenty Three') - -print h.os.release_info -# {'HOME_URL': 'https://fedoraproject.org/', -# 'ID': 'fedora', -# 'NAME': 'Fedora', -# 'PRETTY_NAME': 'Fedora 23 (Workstation Edition)', -# 'VARIANT': 'Workstation Edition', -# 'VARIANT_ID': 'workstation', -# 'VERSION': '23 (Workstation Edition)', -# 'VERSION_ID': '23', -# ... -# } - -print h.os.release_str -# Fedora release 23 (Twenty Three) -``` - -### Storage Management -It is in PROGRESS state. Planed are NFS & LVM services. - -## Requires -* paramiko -* netaddr - -### Power Management -Give you possibility to control host power state, you can restart, poweron, -poweroff host and get host power status. - -Implemented managements: - - * SSH - * IPMI - -```python -ipmi_user = User(pm_user, pm_password) -ipmi_params = { - 'pm_if_type': 'lan', - 'pm_address': 'test-mgmt.testdomain', - 'user': ipmi_user -} -h.add_power_manager( - power_manager.IPMI_TYPE, **ipmi_params -) -# restart host via ipmitool -h.power_manager.restart() -``` - -## Install -```python -python setup.py devop -``` - -[travisimg]: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master -[travis]: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt -[codecovimg]: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt/branch/master/graph/badge.svg -[codecov]: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt -[codehealthimg]: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master/landscape.svg?style=flat -[codehealth]: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..c394baa --- /dev/null +++ b/README.rst @@ -0,0 +1,183 @@ +|Build Status| +|Code Coverage| +|Code Health| + +python-rrmngmnt +=============== + +Remote Resources MaNaGeMeNT + +Intro +----- + +This tool helps you manage remote machines and services running on that. +It is targeted to Linux based machines. All is done via SSH connection, +that means SSH server must be running there already. + +.. code:: python + + from rrmngmnt import Host, RootUser + + h = Host("10.11.12.13") + h.users.append(RootUser('123456')) + exec = h.executor() + print exec.run_cmd(['echo', 'Hello World']) + +Features +-------- + +List of provided interfaces to manage resources on machine, and +examples. + +Filesystem +~~~~~~~~~~ + +Basic file operations, you can find there subset of python 'os' module +related to files. + +.. code:: python + + print h.fs.exists("/path/to/file") + h.fs.chown("/path/to/file", "root", "root") + h.fs.chmod("/path/to/file", "644") + h.fs.unlink("/path/to/file") + +Network +~~~~~~~ + +It allows to manage network configuration. + +.. code:: python + + print h.network.hostname + h.network.hostname = "my.machine.org" + print h.network.all_interfaces() + print h.network.list_bridges() + +Package Management +~~~~~~~~~~~~~~~~~~ + +It encapsulates various package managements. It is able to determine +which package management to use. You can still specify package management +explicitly. + + +Implemented managements: + +- APT +- YUM +- DNF +- RPM + +.. code:: python + + # install htop package using implicit management + h.package_management.install('htop') + # remove htop package using rpm explicitly + h.package_management('rpm').remove('htop') + +System Services +~~~~~~~~~~~~~~~ + +You can toggle system services, it encapsulates various service managements. +It is able to determine which service management to use in most cases. + + +Implemented managements: + +- Systemd +- SysVinit +- InitCtl + +.. code:: python + + if h.service('httpd').status(): + h.service('httpd').stop() + if h.service('httpd').is_enabled(): + h.service('httpd').disable() + +Operating System Info +~~~~~~~~~~~~~~~~~~~~~ + +Host provide ``os`` attribute which allows obtain basic operating +system info. +Note that ``os.release_info`` depends on systemd init system. + +.. code:: python + + print h.os.distribution + # Distribution(distname='Fedora', version='23', id='Twenty Three') + + print h.os.release_info + # {'HOME_URL': 'https://fedoraproject.org/', + # 'ID': 'fedora', + # 'NAME': 'Fedora', + # 'PRETTY_NAME': 'Fedora 23 (Workstation Edition)', + # 'VARIANT': 'Workstation Edition', + # 'VARIANT_ID': 'workstation', + # 'VERSION': '23 (Workstation Edition)', + # 'VERSION_ID': '23', + # ... + # } + + print h.os.release_str + # Fedora release 23 (Twenty Three) + +Storage Management +~~~~~~~~~~~~~~~~~~ + +It is in PROGRESS state. Planed are NFS & LVM services. + +Power Management +~~~~~~~~~~~~~~~~ + +Give you possibility to control host power state, you can restart, +poweron, poweroff host and get host power status. + + +Implemented managements: + +- SSH +- IPMI + +.. code:: python + + ipmi_user = User(pm_user, pm_password) + ipmi_params = { + 'pm_if_type': 'lan', + 'pm_address': 'test-mgmt.testdomain', + 'user': ipmi_user + } + h.add_power_manager( + power_manager.IPMI_TYPE, **ipmi_params + ) + # restart host via ipmitool + h.power_manager.restart() + +Requires +-------- + +- paramiko +- netaddr +- six + +Install +------- + +.. code:: sh + + python setup.py devop + +Test +---- + +.. code:: sh + + tox + +.. |Build Status| image:: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master + :target: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt +.. |Code Coverage| image:: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt/branch/master/graph/badge.svg + :target: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt +.. |Code Health| image:: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master/landscape.svg?style=flat + :target: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master diff --git a/setup.cfg b/setup.cfg index 94edc43..1bf1916 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ name = python-rrmngmnt author = Lukas Bednar author-email = lukyn17@gmail.com summary = Tool to manage remote systems and services -description-file = README.md +description-file = README.rst home-page = https://github.com/rhevm-qe-automation/python-rrmngmnt license = GPLv2 classifier = From 493c16cd75f13aa40711a13720d3c3ca243508ad Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 22 Jun 2016 16:47:55 +0200 Subject: [PATCH 07/55] Fixes issue #54: add -f for hostname cmd (#64) --- rrmngmnt/network.py | 2 +- tests/test_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 0bb373a..56ede50 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -64,7 +64,7 @@ def get_hostname(self): :return: hostname :rtype: string """ - rc, out, _ = self._m.runCmd(['hostname']) + rc, out, _ = self._m.runCmd(['hostname', '-f']) if rc: return None return out.strip() diff --git a/tests/test_network.py b/tests/test_network.py index 9fb2c2e..222a8e1 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -235,7 +235,7 @@ class TestHostNameEtc(object): data = { 'which hostnamectl': (1, '', ''), - 'hostname': (0, 'local', ''), + 'hostname -f': (0, 'local', ''), 'hostname something ; sed -i -e /^HOSTNAME/d /etc/sysconfig/network ' '&& echo HOSTNAME=something >> /etc/sysconfig/network': (0, '', ''), } From 0516e8f384015c53c45ee1d98f5778b1f421e4b3 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 22 Jun 2016 16:53:00 +0200 Subject: [PATCH 08/55] Python 3.5 compliance (#66) Fixes #65 --- .travis.yml | 3 ++- setup.cfg | 3 +++ tox.ini | 7 ++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 926b3ab..b6a7fd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,9 @@ python: - "2.6" - "2.7" - "3.4" + - "3.5" install: - - pip install tox codecov + - pip install tox tox-travis codecov script: - tox after_success: diff --git a/setup.cfg b/setup.cfg index 1bf1916..e761aca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,9 +13,12 @@ classifier = License :: OSI Approved :: GNU General Public License v2 (GPLv2) Operating System :: POSIX Programming Language :: Python + Programming Language :: Python :: 2 Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 keywords = remote resource diff --git a/tox.ini b/tox.ini index 7db2330..b0c54fe 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,10 @@ [tox] -envlist = py26,py27,py34,pep8 +envlist = py26,py27,py34,py35,pep8 +[tox:travis] +2.6 = py26, pep8 +2.7 = py27, pep8 +3.4 = py34, pep8 +3.5 = py35, pep8 [testenv] deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt From e5132e758188fa82c85bec9a9a514989e66c4118 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 17 Aug 2016 13:33:19 +0200 Subject: [PATCH 09/55] py26: drop python 2.6 compatibility (#69) --- .travis.yml | 1 - setup.cfg | 1 - tox.ini | 3 +-- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6a7fd3..539b8fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.4" - "3.5" diff --git a/setup.cfg b/setup.cfg index e761aca..4c28f32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ classifier = Operating System :: POSIX Programming Language :: Python Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.4 diff --git a/tox.ini b/tox.ini index b0c54fe..572314a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] -envlist = py26,py27,py34,py35,pep8 +envlist = py27,py34,py35,pep8 [tox:travis] -2.6 = py26, pep8 2.7 = py27, pep8 3.4 = py34, pep8 3.5 = py35, pep8 From 45f9da1ab47cd3e76125e246e1d2dc85b1d333a6 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Wed, 17 Aug 2016 15:02:11 +0300 Subject: [PATCH 10/55] host: make add to inventory thread safe (#68) --- rrmngmnt/host.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index c453e1d..727c9cf 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -6,10 +6,10 @@ import copy import os import socket +import threading import warnings import netaddr - from rrmngmnt import errors from rrmngmnt import power_manager from rrmngmnt import ssh @@ -31,6 +31,7 @@ class Host(Resource): # The purpose of inventory variable is keeping all instances of # interesting resources in single place. inventory = list() + lock = threading.Lock() default_service_providers = [ Systemd, @@ -94,14 +95,15 @@ def add(self): """ Add host to inventory """ - try: - host = self.get(self.ip) - except ValueError: - pass - else: - self.inventory.remove(host) - self.logger.debug("Adding host with ip '%s' to inventory", self.ip) - self.inventory.append(self) + with self.lock: + try: + host = self.get(self.ip) + except ValueError: + pass + else: + self.inventory.remove(host) + self.logger.debug("Adding host with ip '%s' to inventory", self.ip) + self.inventory.append(self) @property def fqdn(self): From 0a9f96e50919bb01cc78ae64c1021dfa8603acf2 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Thu, 18 Aug 2016 18:11:35 +0200 Subject: [PATCH 11/55] Fix issue #71 related to create_script method (#72) --- rrmngmnt/filesystem.py | 4 +++- tests/common.py | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 310dd20..741da87 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -1,5 +1,7 @@ import os +import six + from rrmngmnt import errors from rrmngmnt.service import Service @@ -91,7 +93,7 @@ def create_script(self, content, path): executor = self.host.executor() with executor.session() as session: with session.open_file(path, 'wb') as fh: - fh.write(content) + fh.write(six.b(content)) self.chmod(path=path, mode="+x") def mkdir(self, path): diff --git a/tests/common.py b/tests/common.py index 362da02..04d47f9 100644 --- a/tests/common.py +++ b/tests/common.py @@ -21,6 +21,23 @@ def close(self): six.StringIO.close(self) +class ByteFakeFile(six.BytesIO): + def __init__(self, buf=six.b('')): + six.BytesIO.__init__(self, six.b(buf)) + self.data = None + + def __exit__(self, *args): + self.close() + + def __enter__(self): + return self + + def close(self): + self.seek(0) + self.data = self.read().decode("utf-8", errors="replace") + six.BytesIO.close(self) + + class FakeExecutor(Executor): cmd_to_data = None files_content = {} @@ -64,7 +81,10 @@ def open_file(self, name, mode): raise else: data = '' - data = FakeFile(data) + if len(mode) == 2 and mode[1] == 'b': + data = ByteFakeFile(data) + else: + data = FakeFile(data) if mode[0] == 'w': data.seek(0) self._executor.files_content[name] = data From e8842bff73ff714306f8d55472d348f992eeed57 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Mon, 22 Aug 2016 10:15:15 +0200 Subject: [PATCH 12/55] Move Host.copy_to to filesystem module (#70) Fixes #25 --- README.rst | 17 +++++++++++ rrmngmnt/filesystem.py | 61 ++++++++++++++++++++++++++++++++++++++++ rrmngmnt/host.py | 4 +++ tests/test_filesystem.py | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/README.rst b/README.rst index c394baa..c949844 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,23 @@ related to files. h.fs.chmod("/path/to/file", "644") h.fs.unlink("/path/to/file") +In additional there are methods to fetch / put file from / to remote system +to / from local system. + +.. code:: python + + h.fs.get("/path/to/remote/file", "/path/to/local/file/or/target/dir") + h.fs.put("/path/to/local/file", "/path/to/remote/file/or/target/dir") + +There is one special method which allows transfer file between hosts. + +.. code:: python + + h1.fs.transfer( + "/path/to/file/on/h1", + h2, "/path/to/file/on/h2/or/target/dir", + ) + Network ~~~~~~~ diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 741da87..5e66552 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -132,6 +132,67 @@ def chmod(self, path, mode): """ self._exec_command(['chmod', mode, path]) + def get(self, path_src, path_dst): + """ + Fetch file from Host and store on local system + + :param path_src: path to file on remote system + :type path_src: str + :param path_dst: path to file on local system or directory + :type path_dst: str + :return: path to destination file + :rtype: str + """ + if os.path.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as ss: + with ss.open_file(path_src, 'rb') as rh: + with open(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + + def put(self, path_src, path_dst): + """ + Upload file from local system to Host + + :param path_src: path to file on local system + :type path_src: str + :param path_dst: path to file on remote system or directory + :type path_dst: str + :return: path to destination file + :rtype: str + """ + if self.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as ss: + with open(path_src, 'rb') as rh: + with ss.open_file(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + + def transfer(self, path_src, target_host, path_dst): + """ + Transfer file from one remote system (self) to other + remote system (target_host). + + :param path_src: path to file on local system + :type path_src: str + :param target_host: target system + :type target_host: instance of Host + :param path_dst: path to file on remote system or directory + :type path_dst: str + :return: path to destination file + :rtype: str + """ + if target_host.fs.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as h1s: + with target_host.executor().session() as h2s: + with h1s.open_file(path_src, 'rb') as rh: + with h2s.open_file(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + def wget(self, url, output_file, progress_handler=None): """ Download file on the host from given url diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 727c9cf..21ffe6e 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -261,6 +261,10 @@ def copy_to(self, resource, src, dst, mode=None, ownership=None): :param ownership: file ownership(ex. ('root', 'root')) :type ownership: tuple """ + warnings.warn( + "This method is deprecated and will be removed. " + "Use Host.fs.transfer instead." + ) with resource.executor().session() as resource_session: with self.executor().session() as host_session: with resource_session.open_file(src, 'rb') as resource_file: diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index b5bdb25..0841e0c 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -130,3 +130,54 @@ def test_listdir_two(self): assert self.get_host().fs.listdir('/path/to/two') == [ 'first', 'second', ] + + +class TestFSGetPutFile(object): + data = { + "[ -d /path/to/put_dir ]": (0, "", ""), + } + files = { + "/path/to/get_file": "data of get_file", + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + return Host(ip) + + def test_get(self, tmpdir): + self.get_host().fs.get("/path/to/get_file", str(tmpdir)) + assert tmpdir.join("get_file").read() == "data of get_file" + + def test_put(self, tmpdir): + p = tmpdir.join("put_file") + p.write("data of put_file") + self.get_host().fs.put(str(p), "/path/to/put_dir") + assert self.files[ + '/path/to/put_dir/put_file'].data == "data of put_file" + + +class TestTransfer(object): + data = { + "[ -d /path/to/dest_dir ]": (0, "", ""), + } + files = { + "/path/to/file_to_transfer": "data to transfer", + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + return Host(ip) + + def test_transfer(self): + self.get_host().fs.transfer( + "/path/to/file_to_transfer", self.get_host("1.1.1.2"), + "/path/to/dest_dir", + ) + assert self.files[ + '/path/to/dest_dir/file_to_transfer'].data == "data to transfer" From bb546ea554c142e9eb2c978f632af00cc20f4732 Mon Sep 17 00:00:00 2001 From: Vaclav Kondula Date: Mon, 3 Oct 2016 16:26:19 +0200 Subject: [PATCH 13/55] Modify FileSystem.touch method (#73) This method now expects same parameters as unix touch command Backwards compatibility has been preserved with a deprecated warning --- rrmngmnt/filesystem.py | 22 ++++++++++++++++++++-- tests/test_filesystem.py | 13 ++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 5e66552..22a88d9 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -1,6 +1,7 @@ import os import six +import warnings from rrmngmnt import errors from rrmngmnt.service import Service @@ -52,7 +53,25 @@ def listdir(self, path): ['ls', '-A1', path] )[1].split() - def touch(self, file_name, path): + def touch(self, *args): + """ + Creates files on host + + __author__ = "vkondula" + :param args: Paths of files to create + :type args: list of str + :returns: True when file creation succeeds, False otherwise + :rtype: bool + """ + if len(args) == 2 and self.isdir(args[1]): + warnings.warn( + "This usecase is deprecated and will be removed. " + "Use list of fullpaths instead" + ) + return self._deprecated_touch(args[0], args[1]) + return self.host.run_command(['touch'] + list(args))[0] == 0 + + def _deprecated_touch(self, file_name, path): """ Creates a file on host @@ -62,7 +81,6 @@ def touch(self, file_name, path): :param path: The path under which the file will be created :type path: str :returns: True when file creation succeeds, False otherwise - False otherwise :rtype: bool """ full_path = os.path.join(path, file_name) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 0841e0c..c46e761 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -30,6 +30,9 @@ class TestFilesystem(object): '[ -f /tmp/nofile ]': (1, '', ''), '[ -d /tmp/dir ]': (0, '', ''), '[ -d /tmp/nodir ]': (1, '', ''), + '[ -d /path/to/file1 ]': (1, '', ''), + '[ -d /path/to ]': (0, '', ''), + '[ -d somefile ]': (1, '', ''), 'rm -f /path/to/remove': (0, '', ''), 'rm -f /dir/to/remove': ( 1, '', 'rm: cannot remove ‘.tox/’: Is a directory', @@ -44,7 +47,8 @@ class TestFilesystem(object): 1, '', 'chmod: cannot access ‘/tmp/nofile’: No such file or directory', ), - 'touch /path/to/file': (0, '', ''), + 'touch /path/to/file /path/to/file1': (0, '', ''), + 'touch /path/to/file2': (0, '', ''), 'touch /path/to/nopermission': (1, '', ''), 'ls -A1 /path/to/empty': (0, '\n', ''), 'ls -A1 /path/to/two': (0, 'first\nsecond\n', ''), @@ -113,10 +117,13 @@ def test_chmod_negative(self): assert "No such file or directory" in str(ex_info.value) def test_touch_positive(self): - assert self.get_host().fs.touch('/path/to/file', '') + assert self.get_host().fs.touch('/path/to/file', '/path/to/file1') def test_touch_negative(self): - assert not self.get_host().fs.touch('/path/to/nopermission', '') + assert not self.get_host().fs.touch('/path/to/nopermission') + + def test_backwards_comp_touch(self): + assert self.get_host().fs.touch('file2', '/path/to') def test_touch_wrong_params(self): with pytest.raises(Exception) as ex_info: From 849feff94528f247abb79e7bd23802d4f55f4902 Mon Sep 17 00:00:00 2001 From: Vaclav Kondula Date: Mon, 10 Oct 2016 10:13:00 +0200 Subject: [PATCH 14/55] Add filesystem.MountPoint (#74) You can now mount, remount and unmount devices --- README.rst | 12 ++++ rrmngmnt/errors.py | 45 ++++++++++++ rrmngmnt/filesystem.py | 143 +++++++++++++++++++++++++++++++++++++++ tests/test_filesystem.py | 20 ++++++ 4 files changed, 220 insertions(+) diff --git a/README.rst b/README.rst index c949844..26d6c54 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,18 @@ There is one special method which allows transfer file between hosts. h2, "/path/to/file/on/h2/or/target/dir", ) +You can also mount devices. + +.. code:: python + + with h.fs.mount_point( + '//example.com/share', opts='ro,guest', + fstype='cifs', target='/mnt/netdisk' + ) as mp: + h.fs.listdir(mp.target) # list mounted directory + mp.remount('rw,sync,guest') # remount with different options + h.fs.touch('%s/new_file' % mp.target) # touch file + Network ~~~~~~~ diff --git a/rrmngmnt/errors.py b/rrmngmnt/errors.py index 8391052..afb20aa 100644 --- a/rrmngmnt/errors.py +++ b/rrmngmnt/errors.py @@ -76,3 +76,48 @@ def __str__(self): return "Operation '{0}' is not supported for {1}: {2}".format( self.operation, self.host, self.reason ) + + +class FileSystemError(GeneralResourceError): + pass + + +class MountError(FileSystemError): + def __init__(self, mp): + self.mp = mp + + +class FailCreateTemp(FileSystemError): + pass + + +class MountCommandError(MountError): + def __init__(self, mp, stdout, stderr): + super(MountCommandError, self).__init__(mp) + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + return ( + """ + stdout:{out} + stderr:{err} + {mp} + """.format( + out=self.stdout, + err=self.stderr, + mp=self.mp, + ) + ) + + +class FailToMount(MountCommandError): + pass + + +class FailToUmount(MountCommandError): + pass + + +class FailToRemount(MountCommandError): + pass diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 22a88d9..84f04a8 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -5,6 +5,7 @@ from rrmngmnt import errors from rrmngmnt.service import Service +from rrmngmnt.resource import Resource class FileSystem(Service): @@ -244,3 +245,145 @@ def wget(self, url, output_file, progress_handler=None): "Failed to download file from url {0}".format(url) ) return output_file + + def mktemp(self, template=None, tmpdir=None, directory=False): + """ + Make temporary file + + :param template: template for path, 'X's are replaced + :type template: str + :param tmpdir: where to create file, if not specified + use $TMPDIR if set, else /tmp + :type tmpdir: str + :param directory: create directory instead of a file + :type directory: bool + :return: absolute path to file or None if failed + :rtype: str + """ + cmd = ['mktemp'] + if tmpdir: + cmd.extend(['-p', tmpdir]) + if directory: + cmd.append('-d') + if template: + cmd.append(template) + rc, out, _ = self.host.run_command(cmd) + if rc: + raise errors.FailCreateTemp(cmd) + return out.replace('\n', '') + + def mount_point( + self, source, target=None, fs_type=None, opts=None + ): + return MountPoint( + self, + source=source, + target=target, + fs_type=fs_type, + opts=opts, + ) + + +class MountPoint(Resource): + """ + Class for mounting devices. + """ + def __init__(self, fs, source, target=None, fs_type=None, opts=None): + """ + Mounts source to target mount point + + __author__ = "vkondula" + :param fs: FileSystem object instance + :type fs: FileSystem + :param source: Full path to source + :type source: str + :param target: Path to target directory, if omitted, a temporary + folder is created instead + :type target: str + :param fs_type: File system type + :type fs_type: str + :param opts: Mount options separated by a comma such as: + 'sync,rw,guest' + :type opts: str + """ + super(MountPoint, self).__init__() + self.fs = fs + self.source = source + self.opts = opts + self.fs_type = fs_type + self.target = target + self._tmp = not bool(target) + self._mounted = False + + def __enter__(self): + self.mount() + return self + + def __exit__(self, type_, value, tb): + try: + self.umount() + except errors.MountError as e: + self.logger.error(e) + if not type_: + raise + + def __str__(self): + return ( + """ + Mounting point: + source: {source} + target: {target} + file system: {fs} + options: {opts} + """.format( + source=self.source, + target=self.target or "*tmp*", + fs=self.fs_type or "DEFAULT", + opts=self.opts or "DEFAULT", + ) + ) + + def mount(self): + if self._tmp: + self.target = self.fs.mktemp(directory=True) + cmd = ['mount', '-v'] + if self.fs_type: + cmd.extend(['-t', self.fs_type]) + if self.opts: + cmd.extend(['-o', self.opts]) + cmd.extend([self.source, self.target]) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToMount(self, out, err) + self._mounted = True + + def umount(self, force=True): + cmd = ['umount', '-v'] + if force: + cmd.append('-f') + cmd.append(self.target) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToUmount(self, out, err) + if self._tmp and not self.fs.listdir(self.target): + self.fs.rmdir(self.target) + self._mounted = False + + def remount(self, opts): + """ + Remount disk + + 'remount' option is implicit + :param opts: Mount options separated by a comma such as: + 'sync,rw,guest' + :type opts: str + """ + if not self._mounted: + raise errors.FailToRemount(self, '', 'not mounted!') + cmd = ['mount', '-v'] + cmd.extend(['-o', 'remount,%s' % opts]) + cmd.append(self.target) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToRemount(self, out, err) + self.opts = opts diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index c46e761..1906234 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -52,6 +52,14 @@ class TestFilesystem(object): 'touch /path/to/nopermission': (1, '', ''), 'ls -A1 /path/to/empty': (0, '\n', ''), 'ls -A1 /path/to/two': (0, 'first\nsecond\n', ''), + 'mktemp -d': (0, '/path/to/tmpdir', ''), + 'mount -v -t xfs -o bind,ro /path/to /path/to/tmpdir': (0, '', ''), + 'ls -A1 /path/to/tmpdir': (0, 'first\nsecond\n', ''), + 'mount -v -o remount,bind,rw /path/to/tmpdir': (0, '', ''), + 'umount -v -f /path/to/tmpdir': (0, '', ''), + 'mount -v /not/device /path/to/tmpdir': ( + 32, '', '/not/device is not a block device\n' + ), } files = {} @@ -138,6 +146,18 @@ def test_listdir_two(self): 'first', 'second', ] + def test_mount_point(self): + with self.get_host().fs.mount_point( + '/path/to', opts='bind,ro', fs_type='xfs' + ) as mp: + assert not mp.remount('bind,rw') + + def test_fail_mount(self): + with pytest.raises(errors.MountError) as ex_info: + with self.get_host().fs.mount_point('/not/device'): + pass + assert "is not a block device" in str(ex_info.value) + class TestFSGetPutFile(object): data = { From a5b2751cecc7d343af2cca8411c12de6b972142e Mon Sep 17 00:00:00 2001 From: Vaclav Kondula Date: Fri, 21 Oct 2016 15:29:14 +0200 Subject: [PATCH 15/55] Add host with ipv6 (#75) You can add host using ipv6 Always try to resolve FQDN. Before if you created host with domain name but not fully qualified it was stored as fqdn Add ability to find default ipv6 for gateway. Add ability to find ipv6 by interface --- rrmngmnt/host.py | 6 ++--- rrmngmnt/network.py | 43 ++++++++++++++++++++++++++++++++++- tests/test_host.py | 4 ++-- tests/test_network.py | 53 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 21ffe6e..01cd8ff 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -61,9 +61,7 @@ def __init__(self, ip, service_provider=None): :type service_provider: class which implement SystemService interface """ super(Host, self).__init__() - self._fqdn = None - if not netaddr.valid_ipv4(ip): - self._fqdn = ip + if not netaddr.valid_ipv4(ip) and not netaddr.valid_ipv6(ip): ip = fqdn2ip(ip) self.ip = ip self.users = list() @@ -107,7 +105,7 @@ def add(self): @property def fqdn(self): - return socket.getfqdn(self.ip) if not self._fqdn else self._fqdn + return socket.getfqdn(self.ip) def add_power_manager(self, pm_type, **init_params): """ diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 56ede50..62219dd 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -204,6 +204,24 @@ def find_default_gw(self): return default_gw[0] return None + @keep_session + def find_default_gwv6(self): + """ + Find host default ipv6 gateway + + :return: default gateway + :rtype: string + """ + out = self._cmd(["ip", "-6", "route"]).splitlines() + for i in out: + if re.search("default", i): + default_gw = re.findall( + r'(?<=\s)[0-9a-fA-F:]{3,}(?=\s)', i + ) + if netaddr.valid_ipv6(default_gw[0]): + return default_gw[0] + return None + @keep_session def find_ips(self): """ @@ -260,7 +278,7 @@ def find_int_by_ip(self, ip): @keep_session def find_ip_by_int(self, interface): """ - Find host interface by interface or Bridge name + Find host ipv4 by interface or Bridge name :param interface: interface to get ip from :type interface: string @@ -275,6 +293,29 @@ def find_ip_by_int(self, interface): return interface_ip return None + @keep_session + def find_ipv6_by_int(self, interface): + """ + Find host global ipv6 by interface or Bridge name + + :param interface: interface to get ipv6 from + :type interface: string + :return: IP or None + :rtype: string or None + """ + out = self._cmd(["ip", "-6", "addr", "show", interface]) + for line in out.splitlines(): + if re.search("global", line): + match_ip = re.search( + r'(?<=\s)[0-9a-fA-F:]{3,}(?=/[0-9]{1,3}\s)', + line, + ) + if match_ip: + interface_ip = match_ip.group() + if netaddr.valid_ipv6(interface_ip): + return interface_ip + return None + @keep_session def find_int_by_bridge(self, bridge): """ diff --git a/tests/test_host.py b/tests/test_host.py index 2d8975e..4d45b09 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -37,9 +37,9 @@ class TestHostFqdnIp(object): def test_host_ip(self): h = Host('127.0.0.1') assert h.ip == '127.0.0.1' - assert h.fqdn == 'localhost' + assert 'localhost' in h.fqdn def test_host_fqdn(self): h = Host('localhost') assert h.ip == '127.0.0.1' - assert h.fqdn == 'localhost' + assert 'localhost' in h.fqdn diff --git a/tests/test_network.py b/tests/test_network.py index 222a8e1..d1b1d9a 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -48,6 +48,19 @@ class TestNetwork(object): ), '', ), + 'ip -6 route': ( + 0, + '\n'.join( + [ + 'unreachable ::/96 dev lo metric 1024 error -101', + 'unreachable 2002:a9fe::/32 lo metric 1024 error -101', + 'unreachable 2002:ac10::/28 lo metric 1024 error -101', + 'fe80:52:0::3fe dev eth0 proto static metric 100 ', + 'default via fe80::0:3fe dev eth0 proto static metric 100', + ] + ), + '', + ), 'ip addr': ( 0, ''.join( @@ -115,6 +128,36 @@ class TestNetwork(object): ), '' ), + 'ip addr show eth0': ( + 0, + '\n'.join( + [ + '2: eth0: mtu ...', + 'link/ether 00:1a:4a:01:3f:1c brd ff:ff:ff:ff:ff:ff', + 'inet 10.11.12.84/22 brd 10.11.12.255 scope global', + 'valid_lft 20343sec preferred_lft 20343sec', + 'inet6 2620:52:0::fe01:3f1c/64 scope global dynamic', + 'valid_lft 2591620sec preferred_lft 604420sec', + 'inet6 fe80::4aff:fe01:3f1c/64 scope link ', + 'valid_lft forever preferred_lft forever', + ] + ), + '' + ), + 'ip -6 addr show eth0': ( + 0, + '\n'.join( + [ + '2: eth0: mtu ...', + 'link/ether 00:1a:4a:01:3f:1c brd ff:ff:ff:ff:ff:ff', + 'inet6 2620:52:0::fe01:3f1c/64 scope global dynamic', + 'valid_lft 2591620sec preferred_lft 604420sec', + 'inet6 fe80::4aff:fe01:3f1c/64 scope link ', + 'valid_lft forever preferred_lft forever', + ] + ), + '' + ), 'brctl show | sed -e "/^bridge name/ d" ' '-e \'s/^\\s\\s*\\(\\S\\S*\\)$/CONT:\\1/I\'': ( 0, @@ -201,6 +244,16 @@ def test_get_mac_address_by_ip(self): expected = "44:1e:a1:73:3c:98" assert get_host().network.get_mac_by_ip("10.11.12.83") == expected + def test_find_ip_by_int(self): + assert get_host().network.find_ip_by_int("eth0") == "10.11.12.84" + + def test_find_ipv6_by_int(self): + expected = "2620:52:0::fe01:3f1c" + assert get_host().network.find_ipv6_by_int("eth0") == expected + + def test_find_default_gwv6(self): + assert get_host().network.find_default_gwv6() == "fe80::0:3fe" + def if_up(self): assert get_host().network.if_up("interface") From dda1a5dc297a0dda731dd675b3cd8a5609b2236a Mon Sep 17 00:00:00 2001 From: Alexey Slaykovsky Date: Thu, 24 Nov 2016 13:55:46 +0100 Subject: [PATCH 16/55] Added file flush method for filesystem. (#76) Also fixed coding in all tests. Signed-off-by: Aleksei Slaikovkii --- rrmngmnt/filesystem.py | 12 ++++++++++++ tests/test_common.py | 2 +- tests/test_db.py | 2 +- tests/test_filesystem.py | 6 +++++- tests/test_host.py | 2 +- tests/test_network.py | 2 +- tests/test_os.py | 2 +- tests/test_package_manager.py | 2 +- tests/test_service.py | 2 +- tests/test_user.py | 2 +- 10 files changed, 25 insertions(+), 9 deletions(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 84f04a8..849172a 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -87,6 +87,18 @@ def _deprecated_touch(self, file_name, path): full_path = os.path.join(path, file_name) return self.host.run_command(['touch', full_path])[0] == 0 + def flush_file(self, file_path): + """ + Flushes the file. + + :param file_path: The path of file to flush. + :type file_path: str + :returns: True if truncated, False otherwise + :rtype: bool + """ + cmd = ["truncate", "-s", "0", file_path] + return self.host.run_command(cmd)[0] == 0 + def read_file(self, path): """ Reads a content of a file in a given path diff --git a/tests/test_common.py b/tests/test_common.py index 315a0b7..ed8090d 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest import netaddr from rrmngmnt import common diff --git a/tests/test_db.py b/tests/test_db.py index c05260a..4088cde 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 1906234..f263532 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User @@ -60,6 +60,7 @@ class TestFilesystem(object): 'mount -v /not/device /path/to/tmpdir': ( 32, '', '/not/device is not a block device\n' ), + 'truncate -s 0 /tmp/file_to_flush': (0, '', ''), } files = {} @@ -104,6 +105,9 @@ def test_rmdir_negative(self): def test_read_file(self): assert self.get_host().fs.read_file("/tmp/file") == "data" + def test_flush_file(self): + assert self.get_host().fs.flush_file("/tmp/file_to_flush") + def test_create_sctript(self): data = "echo hello" path = '/tmp/hello.sh' diff --git a/tests/test_host.py b/tests/test_host.py index 4d45b09..2face3e 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, User, RootUser import pytest diff --git a/tests/test_network.py b/tests/test_network.py index d1b1d9a..16de8ae 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, RootUser from .common import FakeExecutor diff --git a/tests/test_os.py b/tests/test_os.py index 1377029..26c4076 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User diff --git a/tests/test_package_manager.py b/tests/test_package_manager.py index c7ff2c1..853d354 100644 --- a/tests/test_package_manager.py +++ b/tests/test_package_manager.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, User from .common import FakeExecutor import rrmngmnt.package_manager as pm diff --git a/tests/test_service.py b/tests/test_service.py index e8f4176..0218f8a 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host from rrmngmnt.service import SysVinit, Systemd, InitCtl from .common import FakeExecutor diff --git a/tests/test_user.py b/tests/test_user.py index baeb552..00cd647 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import User, InternalDomain, ADUser, Domain From 04f76aa38fb93558ff768d9a4842ac2f720a7b42 Mon Sep 17 00:00:00 2001 From: Alexey Slaykovsky Date: Fri, 25 Nov 2016 11:38:50 +0100 Subject: [PATCH 17/55] Added move method to filesystem class. (#77) Signed-off-by: Alexey Slaykovsky --- rrmngmnt/filesystem.py | 14 ++++++++++++++ tests/test_filesystem.py | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 849172a..bffe49c 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -112,6 +112,20 @@ def read_file(self, path): rc, out, _ = self.host.run_command(cmd) return out if not rc else "" + def move(self, source_path, destination_path): + """ + Moves a file or directory from source to destination. + + :param source_path: The source path to move from. + :type source_path: str + :param destination_path: The destination path to move to. + :type destination_path: str + :return: True if there were no errors, False otherwise. + :rtype: bool + """ + cmd = ["mv", source_path, destination_path] + return self.host.run_command(cmd)[0] == 0 + def create_script(self, content, path): """ Create script on filesystem, and make it executable. diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index f263532..b41d20c 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -61,6 +61,7 @@ class TestFilesystem(object): 32, '', '/not/device is not a block device\n' ), 'truncate -s 0 /tmp/file_to_flush': (0, '', ''), + 'mv /tmp/source /tmp/destination': (0, '', ''), } files = {} @@ -105,6 +106,9 @@ def test_rmdir_negative(self): def test_read_file(self): assert self.get_host().fs.read_file("/tmp/file") == "data" + def test_move(self): + assert self.get_host().fs.move("/tmp/source", "/tmp/destination") + def test_flush_file(self): assert self.get_host().fs.flush_file("/tmp/file_to_flush") From 250d3a003ffd465c4ce6cf5a3e767bfca6889f87 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Thu, 1 Dec 2016 16:12:06 +0100 Subject: [PATCH 18/55] setup.cfg: set default sdist format (#78) --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 4c28f32..528a101 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,3 +34,5 @@ build_requires = python python-setuptools python-pbr +[sdist] +formats = zip,gztar From 7526857474c585caf82a0fa02f47b4176a394e99 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Tue, 17 Jan 2017 21:40:42 +0200 Subject: [PATCH 19/55] Service: Don't fail when service have @ in the name Start service fail when service have @ in the name. For example "serial-getty@ttyS0' is not in unit-files but "serial-getty@" is there. --- rrmngmnt/service.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/service.py b/rrmngmnt/service.py index 9222678..edb1f95 100644 --- a/rrmngmnt/service.py +++ b/rrmngmnt/service.py @@ -1,4 +1,5 @@ from rrmngmnt.resource import Resource +import re class Service(Resource): @@ -169,6 +170,11 @@ class Systemd(SystemService): def _can_handle(self): super(Systemd, self)._can_handle() + + orig_name = self.name + if "@" in self.name: + self.name = re.match(r'^.*@', self.name).group(0) + cmd = ( 'systemctl', 'list-unit-files', '|', 'grep', '-o', '^[^.][^.]*.service', '|', @@ -180,8 +186,9 @@ def _can_handle(self): out = out.strip().splitlines() if rc or self.name not in out: raise self.CanNotHandle( - "%s is not listed in %s" % (self.name, out) + "%s is not listed in %s" % (orig_name, out) ) + self.name = orig_name def _execute(self, action): cmd = [ From 601c0cffd2f2a77d1a039355eaf4fbb0f3fc9db4 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Mon, 13 Feb 2017 17:59:56 +0200 Subject: [PATCH 20/55] power_manager: add reasonable timeout for SSH actions (#80) Sometime the code can stuck, because of command like `poweroff -f`, so we need to add reasonable timeout for TCP and IO. --- rrmngmnt/power_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/power_manager.py b/rrmngmnt/power_manager.py index 76b031c..a7d68df 100644 --- a/rrmngmnt/power_manager.py +++ b/rrmngmnt/power_manager.py @@ -62,7 +62,9 @@ def _exec_pm_command(self, command, *args): try: t_command = list(command) t_command += args - self.host.executor().run_cmd(t_command) + self.host.executor().run_cmd( + t_command, tcp_timeout=20, io_timeout=20 + ) except socket.timeout as e: self.logger.debug("Socket timeout: %s", e) except Exception as e: From 36b217ef7d04bb1700e6c00557c87e594351b331 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Fri, 31 Mar 2017 12:44:44 +0200 Subject: [PATCH 21/55] Fix test (#83) --- tests/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_common.py b/tests/test_common.py index ed8090d..00b6bfa 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -5,7 +5,7 @@ def test_fqdn2ip_positive(): - ip = common.fqdn2ip('github.org') + ip = common.fqdn2ip('github.com') assert netaddr.valid_ipv4(ip) From 43c0e9ed07e54e1677d27a4fc58c318f88b0171a Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Thu, 27 Apr 2017 10:37:23 +0200 Subject: [PATCH 22/55] Automate process to publish new release (#84) Fix #81 --- .travis.yml | 12 ++++++++++++ setup.cfg | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 539b8fc..949bcd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,3 +9,15 @@ script: - tox after_success: - codecov +deploy: + provider: pypi + user: "lukas-bednar" + password: + secure: "\ + ks4q6t0YBc4i3hr5uYCepUi05SuBfkA6l2vakuqcQunuwClaCN3ryP5aKCKk3673wdKBh2ee\ + L+VrKrmEnyRTrgo+t02ODSibAMeytwq254m526FiUbATemNrDyPtv7XTO/Yp9yFPwHbpoH8b\ + dTa4MhTUm6qXcRtRdYvfU8zVKUU=" + on: + branch: master + tags: true + skip_upload_docs: true diff --git a/setup.cfg b/setup.cfg index 528a101..3987576 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,4 +35,4 @@ build_requires = python-setuptools python-pbr [sdist] -formats = zip,gztar +formats = zip From 63fe633cc6bc96af5791b568cb57b1e260c33284 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Wed, 3 May 2017 10:45:24 +0300 Subject: [PATCH 23/55] Convert docstrings to google style (#86) --- rrmngmnt/common.py | 9 +- rrmngmnt/db.py | 27 +++-- rrmngmnt/errors.py | 23 ++-- rrmngmnt/executor.py | 11 +- rrmngmnt/filesystem.py | 194 ++++++++++++++--------------- rrmngmnt/host.py | 181 ++++++++++++++------------- rrmngmnt/network.py | 235 +++++++++++++++++++----------------- rrmngmnt/operatingsystem.py | 54 +++++---- rrmngmnt/package_manager.py | 76 +++++++----- rrmngmnt/power_manager.py | 10 +- rrmngmnt/service.py | 3 +- rrmngmnt/ssh.py | 63 +++++----- rrmngmnt/storage.py | 69 +++++------ rrmngmnt/user.py | 27 ++--- 14 files changed, 507 insertions(+), 475 deletions(-) diff --git a/rrmngmnt/common.py b/rrmngmnt/common.py index 4845f6f..3e56e68 100644 --- a/rrmngmnt/common.py +++ b/rrmngmnt/common.py @@ -5,10 +5,11 @@ def fqdn2ip(fqdn): """ translate fqdn to IP - :param fqdn: host name - :type fqdn: string - :return: IP - :rtype: string + Args: + fqdn (str): host name + + Returns: + str: IP address """ try: return socket.gethostbyname(fqdn) diff --git a/rrmngmnt/db.py b/rrmngmnt/db.py index 57501cd..d7ff163 100644 --- a/rrmngmnt/db.py +++ b/rrmngmnt/db.py @@ -5,12 +5,10 @@ class Database(Service): def __init__(self, host, name, user): """ - :param host: Remote resouce to DB machine - :type host: instance of Host - :param name: database name - :type name: str - :param user: user/role - :type user: instance of User + Args: + host (Host): Remote resouce to DB machine + name (str): database name + user (User): user/role """ super(Database, self).__init__(host) self.name = name @@ -20,12 +18,12 @@ def psql(self, sql, *args): """ Execute psql command on host - :param sql: sql command - :type sql: string - :param args: positional format arguments for command - :type args: list of arguments - :return: list of lines with records - :rtype: list(list(string, string, ...)) + Args: + sql (str): sql command + args (list): positional format arguments for command + + Returns: + list: list of lines with records. """ separator = '__RECORD_SEPARATOR__' sql = sql % tuple(args) @@ -48,7 +46,10 @@ def psql(self, sql, *args): ] # NOTE: I am considering to use Psycopg2 to access DB directly. # I need to think whether it is better or not. - # We need to realize that connection can be forbiden from outside ... + # We need to realize that connection can be forbidden from outside ... def restart(self): + """ + Restart postgresql service + """ self.host.service('postgresql').restart() diff --git a/rrmngmnt/errors.py b/rrmngmnt/errors.py index afb20aa..8854487 100644 --- a/rrmngmnt/errors.py +++ b/rrmngmnt/errors.py @@ -10,14 +10,11 @@ class CommandExecutionFailure(GeneralResourceError): """ def __init__(self, executor, cmd, rc, err): """ - :param executor: executor used for execution - :type executor: instance of RemoteExecutor - :param cmd: executed command - :type cmd: list - :param rc: return code - :type rc: int - :param err: standard error output if provided - :type err: string + Args: + executor (RemoteExecutor): executor used for execution + cmd (list): executed command + rc (int): return code + err (str): standard error output if provided """ super(CommandExecutionFailure, self).__init__(executor, cmd, rc, err) @@ -51,12 +48,10 @@ class UnsupportedOperation(GeneralResourceError): """ def __init__(self, host, operation, reason): """ - :param host: relevant host - :type host: instance of Host - :param operation: name of unsupported operation - :type operation: str - :param reason: message - :type message: str + Args: + host (Host): relevant host + operation (str): name of unsupported operation + reason (str): message """ super(UnsupportedOperation, self).__init__(host, operation, reason) diff --git a/rrmngmnt/executor.py b/rrmngmnt/executor.py index a9f2251..3534ce7 100644 --- a/rrmngmnt/executor.py +++ b/rrmngmnt/executor.py @@ -81,8 +81,8 @@ def rc(self): def __init__(self, user): """ - :param user: user - :type user: instance of user + Args: + user (User): user """ super(Executor, self).__init__() self.user = user @@ -92,10 +92,9 @@ def session(self): def run_cmd(self, cmd, input_=None): """ - :param cmd: command - :type cmd: list - :param input_: input data - :type input_: str + Args: + cmd (list): command + input_(str): input data """ with self.session() as session: return session.run_cmd(cmd, input_) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index bffe49c..e46734d 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -59,10 +59,12 @@ def touch(self, *args): Creates files on host __author__ = "vkondula" - :param args: Paths of files to create - :type args: list of str - :returns: True when file creation succeeds, False otherwise - :rtype: bool + + Args: + args (list): Paths of files to create + + Returns: + bool: True when file creation succeeds, False otherwise """ if len(args) == 2 and self.isdir(args[1]): warnings.warn( @@ -77,12 +79,13 @@ def _deprecated_touch(self, file_name, path): Creates a file on host __author__ = "ratamir" - :param file_name: The file to create - :type file_name: str - :param path: The path under which the file will be created - :type path: str - :returns: True when file creation succeeds, False otherwise - :rtype: bool + + Args: + file_name (str): The file to create + path (str): The path under which the file will be created + + Returns: + bool: True when file creation succeeds, False otherwise """ full_path = os.path.join(path, file_name) return self.host.run_command(['touch', full_path])[0] == 0 @@ -91,10 +94,11 @@ def flush_file(self, file_path): """ Flushes the file. - :param file_path: The path of file to flush. - :type file_path: str - :returns: True if truncated, False otherwise - :rtype: bool + Args: + file_path (str): The path of file to flush. + + Returns: + bool: True if truncated, False otherwise """ cmd = ["truncate", "-s", "0", file_path] return self.host.run_command(cmd)[0] == 0 @@ -103,10 +107,11 @@ def read_file(self, path): """ Reads a content of a file in a given path - :param path: The path from where to take a content from - :type path: str - :return: Content of a file - :rtype: str + Args: + path (str): The path from where to take a content from + + Returns: + str: Content of a file """ cmd = ["cat", path] rc, out, _ = self.host.run_command(cmd) @@ -116,12 +121,12 @@ def move(self, source_path, destination_path): """ Moves a file or directory from source to destination. - :param source_path: The source path to move from. - :type source_path: str - :param destination_path: The destination path to move to. - :type destination_path: str - :return: True if there were no errors, False otherwise. - :rtype: bool + Args: + source_path (str): The source path to move from. + destination_path (str): The destination path to move to. + + Returns: + bool: True if there were no errors, False otherwise. """ cmd = ["mv", source_path, destination_path] return self.host.run_command(cmd)[0] == 0 @@ -130,10 +135,9 @@ def create_script(self, content, path): """ Create script on filesystem, and make it executable. - :param content: content of the script - :type content: str - :param path: path to script to create - :type path: str + Args: + content (str): content of the script + path (str): path to script to create """ executor = self.host.executor() with executor.session() as session: @@ -145,9 +149,11 @@ def mkdir(self, path): """ Create directory on host - :param path: directory path - :type path: str - :raises: CommandExecutionFailure, if mkdir failed + Args: + path (str): directory path + + Raises: + CommandExecutionFailure: If mkdir failed """ self._exec_command(['mkdir', path]) @@ -155,13 +161,13 @@ def chown(self, path, username, groupname): """ Change owner of file or directory - :param path: file or directory path - :type path: str - :param username: change user owner to username - :type username: str - :param groupname: change group owner to groupname - :type groupname: str - :raises: CommandExecutionFailure, if chown failed + Args: + path (str): file or directory path + username (str): change user owner to username + groupname (str): change group owner to groupname + + Raises: + CommandExecutionFailure: If chown failed """ self._exec_command(['chown', '%s:%s' % (username, groupname), path]) @@ -169,11 +175,12 @@ def chmod(self, path, mode): """ Change permission of directory or file - :param path: file or directory path - :type path: str - :param mode: permission mode(600 for example or u+x) - :type mode: str - :raises: CommandExecutionFailure, if chmod failed + Args: + path (str): file or directory path + mode (str): permission mode(600 for example or u+x) + + Raises: + CommandExecutionFailure: If chmod failed """ self._exec_command(['chmod', mode, path]) @@ -181,12 +188,12 @@ def get(self, path_src, path_dst): """ Fetch file from Host and store on local system - :param path_src: path to file on remote system - :type path_src: str - :param path_dst: path to file on local system or directory - :type path_dst: str - :return: path to destination file - :rtype: str + Args: + path_src (str): path to file on remote system + path_dst (str): path to file on local system or directory + + Returns: + str: Path to destination file """ if os.path.isdir(path_dst): path_dst = os.path.join(path_dst, os.path.basename(path_src)) @@ -200,12 +207,12 @@ def put(self, path_src, path_dst): """ Upload file from local system to Host - :param path_src: path to file on local system - :type path_src: str - :param path_dst: path to file on remote system or directory - :type path_dst: str - :return: path to destination file - :rtype: str + Args: + path_src (str): path to file on local system + path_dst (str): path to file on remote system or directory + + Returns: + str: path to destination file """ if self.isdir(path_dst): path_dst = os.path.join(path_dst, os.path.basename(path_src)) @@ -220,14 +227,13 @@ def transfer(self, path_src, target_host, path_dst): Transfer file from one remote system (self) to other remote system (target_host). - :param path_src: path to file on local system - :type path_src: str - :param target_host: target system - :type target_host: instance of Host - :param path_dst: path to file on remote system or directory - :type path_dst: str - :return: path to destination file - :rtype: str + Args: + path_src (str): path to file on local system + target_host (Host): target system + path_dst (str): path to file on remote system or directory + + Returns: + str: path to destination file """ if target_host.fs.isdir(path_dst): path_dst = os.path.join(path_dst, os.path.basename(path_src)) @@ -242,14 +248,13 @@ def wget(self, url, output_file, progress_handler=None): """ Download file on the host from given url - :param url: url to file - :type url: str - :param output_file: full path to output file - :type output_file: str - :param progress_handler: progress handler function - :type progress_handler: func - :return: absolute path to file - :rtype: str + Args: + url (str): url to file + output_file (str): full path to output file + progress_handler (func): progress handler function + + Returns: + str: absolute path to file """ rc = None host_executor = self.host.executor() @@ -276,15 +281,14 @@ def mktemp(self, template=None, tmpdir=None, directory=False): """ Make temporary file - :param template: template for path, 'X's are replaced - :type template: str - :param tmpdir: where to create file, if not specified - use $TMPDIR if set, else /tmp - :type tmpdir: str - :param directory: create directory instead of a file - :type directory: bool - :return: absolute path to file or None if failed - :rtype: str + Args: + template (str): template for path, 'X's are replaced + tmpdir (str): where to create file, if not specified + use $TMPDIR if set, else /tmp + directory (bool): create directory instead of a file + + Returns: + str: absolute path to file or None if failed """ cmd = ['mktemp'] if tmpdir: @@ -319,18 +323,15 @@ def __init__(self, fs, source, target=None, fs_type=None, opts=None): Mounts source to target mount point __author__ = "vkondula" - :param fs: FileSystem object instance - :type fs: FileSystem - :param source: Full path to source - :type source: str - :param target: Path to target directory, if omitted, a temporary - folder is created instead - :type target: str - :param fs_type: File system type - :type fs_type: str - :param opts: Mount options separated by a comma such as: - 'sync,rw,guest' - :type opts: str + + Args: + fs (FileSystem): FileSystem object instance + source (str): Full path to source + target (str): Path to target directory, if omitted, a temporary + folder is created instead + fs_type (str): File system type + opts (str): Mount options separated by a comma such as: + 'sync,rw,guest' """ super(MountPoint, self).__init__() self.fs = fs @@ -397,12 +398,11 @@ def umount(self, force=True): def remount(self, opts): """ - Remount disk + Remount disk. 'remount' option is implicit - 'remount' option is implicit - :param opts: Mount options separated by a comma such as: - 'sync,rw,guest' - :type opts: str + Args: + opts (str): Mount options separated by a comma such as: + 'sync,rw,guest' """ if not self._mounted: raise errors.FailToRemount(self, '', 'not mounted!') diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 01cd8ff..4d58613 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -55,10 +55,9 @@ def process(self, msg, kwargs): def __init__(self, ip, service_provider=None): """ - :param ip: IP address of machine or resolvable FQDN - :type ip: string - :param service_provider: system service handler - :type service_provider: class which implement SystemService interface + Args: + ip (str): IP address of machine or resolvable FQDN + service_provider (Service): system service handler """ super(Host, self).__init__() if not netaddr.valid_ipv4(ip) and not netaddr.valid_ipv6(ip): @@ -79,10 +78,12 @@ def __str__(self): def get(cls, ip): """ Get host from inventory - :param ip: IP address of machine or resolvable FQDN - :type ip: str - :return: host - :rtype: Host + + Args: + ip (str): IP address of machine or resolvable FQDN + + Returns: + Host: host instance """ host = [h for h in cls.inventory if h.ip == ip or h.fqdn == ip] if not host: @@ -111,10 +112,10 @@ def add_power_manager(self, pm_type, **init_params): """ Add power power manager to host - :param pm_type: power manager type(power_manager.SSH_TYPE for example) - :type pm_type: str - :param init_params: power manager init parameters - :type init_params: dict + Args: + pm_type (str): power manager type + (power_manager.SSH_TYPE for example) + init_params (dict): power manager init parameters """ self._power_managers[pm_type] = getattr( power_manager, power_manager.MANAGERS[pm_type] @@ -124,11 +125,15 @@ def get_power_manager(self, pm_type=None): """ Get host power manager - :param pm_type: power manager type(power_manager.SSH_TYPE for example) - :type pm_type: str - :return: instance of PowerManager - :rtype: PowerManager - :raises: Exception + Args: + pm_type (str): power manager type(power_manager.SSH_TYPE for + example) + + Returns: + PowerManager: instance of powermanager + + Raises: + Exception: If power manager not supported """ if self._power_managers: if pm_type: @@ -154,8 +159,8 @@ def add_user(self, user): """ Adds user to users collection, and tries remove duplicities. - :param user: user to add - :type user: instance of rrmngmnt.User + Args: + user (User): user to add """ for u in self.users[:]: if user.get_full_name() == u.get_full_name(): @@ -165,11 +170,10 @@ def add_user(self, user): def _set_executor_user(self, user): """ This method explicitly set user which is used to execute commands - on host. - And adds user into users collection. + on host. And adds user into users collection. - :param user: specific user - :type user: instance of rrmngmnt.User + Args: + user (User): specific user """ self._executor_user = user self.add_user(user) @@ -178,8 +182,8 @@ def _get_executor_user(self): """ The user which is supposed to be used for command execution. - :return: user - :rtype: instance of rrmngmnt.User + Returns: + user: instance of User """ if self._executor_user: return copy.copy(self._executor_user) @@ -207,9 +211,10 @@ def executor(self, user=None, pkey=False): """ Gives you executor to allowing command execution - :param user: the executed commands will be executed under this user. - when it is None, the default executor user is used, - see set_executor_user method for more info. + Args: + user (User): the executed commands will be executed under this + user. when it is None, the default executor user is used, + see set_executor_user method for more info. """ if user is None: user = self.executor_user @@ -222,16 +227,14 @@ def run_command( """ Run command on host - :param command: command - :type command: list - :param input_: input data - :type input_: str - :param tcp_timeout: tcp timeout - :type tcp_timeout: float - :param io_timeout: timeout for data operation (read/write) - :type io_timeout: float - :return: tuple of (rc, out, err) - :rtype: tuple + Args: + command (list): command + input_ (str): input data + tcp_timeout (float): tcp timeout + `io_timeout (float): timeout for data operation (read/write) + + Returns: + tuple: tuple of (rc, out, err) """ self.logger.info("Executing command %s", ' '.join(command)) rc, out, err = self.executor(user=user, pkey=pkey).run_cmd( @@ -248,16 +251,12 @@ def copy_to(self, resource, src, dst, mode=None, ownership=None): """ Copy to host from another resource - :param resource: resource to copy from - :type resource: instance of Host - :param src: path to source - :type src: str - :param dst: path to destination - :type dst: str - :param mode: file permissions - :type mode: str - :param ownership: file ownership(ex. ('root', 'root')) - :type ownership: tuple + Args: + src (str): Path to source + dst (str): Path to destination + resource (instance of Host): Resource to copy from + mode (str): File permissions + ownership (tuple): File ownership(ex. ('root', 'root')) """ warnings.warn( "This method is deprecated and will be removed. " @@ -298,12 +297,13 @@ def service(self, name, timeout=None): """ Create service provider for desired service - :param name: service name - :type name: string - :param timeout: expected time to complete operations - :type timeout: int - :return: service provider for desired service - :rtype: instance of SystemService + :Args: + name (string): Service name + timeout (int): Expected time to complete operations + + Returns: + instance of SystemService: Service provider for desired service + """ if self._service_provider is None: # we need to pick up service provider, @@ -325,10 +325,13 @@ def get_ssh_public_key(self, user=None): """ Get SSH public key - :param user: what user to get ssh keys for, default is root - :type user: instance of rrmngmnt.User - :return: SSH public key - :rtype: str + Args: + user (instance of rrmngmnt.User): What user to get ssh keys for, + default is root + + Returns: + str: Ssh public key + """ if user is None: user = copy.copy(self.root_user) @@ -355,12 +358,13 @@ def remove_remote_host_ssh_key(self, remote_host, user=None): """ Remove remote host keys (ip, fqdn) from KNOWN_HOSTS file - :param remote_host: Remote host resource object - :type remote_host: Host - :param user: what user to remove ssh keys for, default is root - :type user: instance of rrmngmnt.User - :return: True/False - :rtype: bool + Args: + remote_host (Host): Remote host resource object + user (instance of rrmngmnt.User): What user to remove ssh keys for, + default is root + + Returns: + bool: True/false """ if user is None: user = copy.copy(self.root_user) @@ -380,10 +384,12 @@ def remove_remote_key_from_authorized_keys(self, user=None): """ Remove remote ssh key from AUTHORIZED_KEYS file - :param user: what user to remove from authorized_keys, default is root - :type user: instance of rrmngmnt.User - :return: True/False - :rtype: bool + Args: + user (instance of rrmngmnt.User): What user to remove from + authorized_keys, default is root + + Returns: + bool: True/false """ if user is None: user = copy.copy(self.root_user) @@ -401,12 +407,15 @@ def get_os_info(self): """ Get OS info (Distro, version and code name) - :return: Results {dist: , ver: , name:} - example: - {'dist': 'Red Hat Enterprise Linux Server', + Returns: + dict: Results {dist: , ver: , name:} + + Examples: + { + 'dist': 'Red Hat Enterprise Linux Server', 'name': 'Maipo', - 'ver': '7.1'} - :rtype: dict + 'ver': '7.1' + } """ warnings.warn( "This method is deprecated and will be removed. " @@ -455,15 +464,14 @@ def create_script( """ Create script on resource - :param content: content of the script - :type content: str - :param name_of_script: name of script to create - :type name_of_script: str - :param destination_path: directory on host to copy script - :type destination_path: str - :returns: Script absolute path, if creation success, - otherwise empty string - :rtype: str + Args: + content (str): Content of the script + name_of_script (str): Name of script to create + destination_path (str): Directory on host to copy script + + Returns: + str: Script absolute path, if creation success, otherwise empty + string """ warnings.warn( "This method is deprecated and will be removed. " @@ -480,10 +488,11 @@ def is_connective(self, tcp_timeout=20.0): """ Check if host is connective via ssh - :param tcp_timeout: time to wait for response - :type tcp_timeout: float - :return: True if host is connective, False otherwise - :rtype: bool + Args: + tcp_timeout (float): Time to wait for response + + Returns: + bool: True if host is connective, false otherwise """ warnings.warn( "This method is deprecated and will be removed. " diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 62219dd..f005dbf 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -61,8 +61,9 @@ def __init__(self, session): def get_hostname(self): """ Get hostname - :return: hostname - :rtype: string + + Returns: + str: Hostname """ rc, out, _ = self._m.runCmd(['hostname', '-f']) if rc: @@ -73,8 +74,9 @@ def get_hostname(self): def set_hostname(self, name): """ Set hostname persistently - :param name: hostname to be set - :type name: string + + Args: + name (str): Hostname to be set """ net_config = '/etc/sysconfig/network' cmd = [ @@ -99,8 +101,9 @@ class HostnameCtlHandler(HostnameHandler): def get_hostname(self): """ Get hostname - :return: hostname - :rtype: string + + Returns: + str: Hostname """ cmd = [ 'hostnamectl', 'status', '|', @@ -117,8 +120,9 @@ def get_hostname(self): def set_hostname(self, name): """ Set hostname persistently - :param name: hostname to be set - :type name: string + + Args: + name (str): Hostname to be set """ cmd = ['hostnamectl', 'set-hostname', name] rc, _, err = self._m.runCmd(cmd) @@ -177,8 +181,8 @@ def all_interfaces(self): """ Lists interfaces - :return: list of interfaces - :rtype: list of strings + Returns: + list of strings: List of interfaces """ out = self._cmd( "ls -la /sys/class/net | grep 'dummy_\|pci' | grep -o '[" @@ -193,8 +197,8 @@ def find_default_gw(self): """ Find host default gateway - :return: default gateway - :rtype: string + Returns: + str: Default gateway """ out = self._cmd(["ip", "route"]).splitlines() for i in out: @@ -209,8 +213,8 @@ def find_default_gwv6(self): """ Find host default ipv6 gateway - :return: default gateway - :rtype: string + Returns: + str: Default gateway """ out = self._cmd(["ip", "-6", "route"]).splitlines() for i in out: @@ -227,8 +231,9 @@ def find_ips(self): """ Find host IPs - :return: list of ips and list of cird ips - :rtype: tuple(list of strings, list of strings) + Returns: + tuple(list of strings, list of strings): List of ips and list of + cird ips """ ips = [] ip_and_netmask = [] @@ -247,12 +252,13 @@ def find_ip_by_default_gw(self, default_gw, ips_and_mask): """ Find IP by default gateway - :param default_gw: default gw of the host - :type default_gw: string - :param ips_and_mask: list of host ips with mask x.x.x.x/xx - :type ips_and_mask: list of strings - :return: ip - :rtype: string + Args: + ips_and_mask (list of strings): List of host ips with + mask x.x.x.x/xx + default_gw (str): Default gw of the host + + Returns: + str: Ip """ dgw = netaddr.IPAddress(default_gw) for ip_mask in ips_and_mask: @@ -267,10 +273,11 @@ def find_int_by_ip(self, ip): """ Find host interface or bridge by IP - :param ip: ip of the interface to find - :type ip: string - :return: interface - :rtype: string + Args: + ip (str): Ip of the interface to find + + Returns: + str: Interface """ out = self._cmd(["ip", "addr", "show", "to", ip]) return out.split(":")[1].strip() @@ -280,10 +287,11 @@ def find_ip_by_int(self, interface): """ Find host ipv4 by interface or Bridge name - :param interface: interface to get ip from - :type interface: string - :return: IP or None - :rtype: string or None + Args: + interface (str): Interface to get ip from + + Returns: + str or None: Ip or none """ out = self._cmd(["ip", "addr", "show", interface]) match_ip = re.search(r'[0-9]+(?:\.[0-9]+){3}', out) @@ -298,10 +306,11 @@ def find_ipv6_by_int(self, interface): """ Find host global ipv6 by interface or Bridge name - :param interface: interface to get ipv6 from - :type interface: string - :return: IP or None - :rtype: string or None + Args: + interface (str): Interface to get ipv6 from + + Returns: + str or None: Ip or none """ out = self._cmd(["ip", "-6", "addr", "show", interface]) for line in out.splitlines(): @@ -321,10 +330,11 @@ def find_int_by_bridge(self, bridge): """ Find host interface by Bridge name - :param bridge: bridge to get ip from - :type bridge: string - :return: interface - :rtype: string + Args: + bridge (str): Bridge to get ip from + + Returns: + str: Interface """ bridge = self.get_bridge(bridge) try: @@ -340,10 +350,11 @@ def find_mac_by_int(self, interfaces): """ Find interfaces MAC by interface name - :param interfaces: list of interfaces - :type interfaces: list of strings - :return: list of macs - :rtype: list of strings + Args: + interfaces (list of strings): List of interfaces + + Returns: + list of strings: List of macs """ mac_list = list() for interface in interfaces: @@ -357,11 +368,11 @@ def find_mac_by_int(self, interfaces): @keep_session def find_mgmt_interface(self): """ - Find host mgmt interface (interface with IP that lead - to default gateway) + Find host mgmt interface (interface with IP that lead to default + gateway) - :return: interface - :rtype: string + Returns: + str: Interface """ host_ip = self.find_ips() host_dg = self.find_default_gw() @@ -374,8 +385,8 @@ def list_bridges(self): """ List of bridges on host - :return: list of bridges - :rtype: list of dict(name, id, stp, interfaces) + Returns: + list of dict(name, id, stp, interfaces): List of bridges """ bridges = [] cmd = [ @@ -409,8 +420,8 @@ def get_bridge(self, name): """ Find bridge by name - :return: bridge - :rtype: dict(name, id, stp, interfaces) + Returns: + dict(name, id, stp, interfaces): Bridge """ bridges = [ bridge for bridge in self.list_bridges() @@ -425,12 +436,12 @@ def add_bridge(self, bridge, network): """ Add bridge and add network to the bridge on host - :param bridge: Bridge name - :type bridge: str - :param network: Network name - :type network: str - :return: True/False - :rtype: bool + Args: + bridge (str): Bridge name + network (str): Network name + + Returns: + bool: True/false """ cmd_add_br = ["brctl", "addbr", bridge] cmd_add_if = ["brctl", "addif", bridge, network] @@ -443,10 +454,11 @@ def delete_bridge(self, bridge): """ Add bridge and add network to the bridge on host - :param bridge: Bridge name - :type bridge: str - :return: True/False - :rtype: bool + Args: + bridge (str): Bridge name + + Returns: + bool: True/false """ cmd_br_down = ["ip", "link", "set", "down", bridge] cmd_del_br = ["brctl", "delbr", bridge] @@ -459,8 +471,8 @@ def get_info(self): """ Get network info for host, return info for main IP. - :return: network info - :rtype: dict + Returns: + dict: Network info """ net_info = {} gateway = self.find_default_gw() @@ -493,12 +505,10 @@ def create_ifcfg_file(self, nic, params, ifcfg_path=IFCFG_PATH): """ Create ifcfg file - :param nic: NIC name - :type nic: str - :param params: Ifcfg file content - :type params: dict - :param ifcfg_path: Ifcfg files path - :type ifcfg_path: str + Args: + nic (str): Nic name + ifcfg_path (str): Ifcfg files path + params (dict): Ifcfg file content """ dst = os.path.join(ifcfg_path, "ifcfg-%s" % nic) self.logger.info("Creating %s on %s", dst, self.host.fqdn) @@ -512,12 +522,12 @@ def delete_ifcfg_file(self, nic, ifcfg_path=IFCFG_PATH): """ Delete ifcfg file - :param nic: NIC name - :type nic: str - :param ifcfg_path: Ifcfg files path - :type ifcfg_path: str - :return: True/False - :rtype: bool + Args: + nic (str): Nic name + ifcfg_path (str): Ifcfg files path + + Returns: + bool: True/false """ dst = os.path.join(ifcfg_path, "ifcfg-%s" % nic) logger.info("Delete %s ", dst) @@ -530,16 +540,14 @@ def send_icmp(self, dst, count="5", size="1500", extra_args=None): """ Send ICMP to destination IP/FQDN - :param dst: IP/FQDN to send ICMP to - :type dst: str - :param count: Number of ICMP packets to send - :type count: str - :param size: Size of the ICMP packet - :type size: str - :param extra_args: Extra args for ping command - :type extra_args: str - :return: True/False - :rtype: bool + Args: + count (str): Number of icmp packets to send + extra_args (str): Extra args for ping command + dst (str): Ip/fqdn to send icmp to + size (str): Size of the icmp packet + + Returns: + bool: True/false """ cmd = ["ping", dst, "-c", count, "-s", size] if size != "1500": @@ -558,12 +566,12 @@ def set_mtu(self, nics, mtu="1500"): """ Set MTU on NICs - :param nics: List on NICs - :type nics: list - :param mtu: MTU size - :type mtu: str - :return: True or raise Exception - :rtype: bool or Exception + Args: + nics (list): List on nics + mtu (str): Mtu size + + Returns: + bool or Exception: True or raise exception """ base_cmd = "ip link set mtu %s %s" for nic in nics: @@ -575,10 +583,11 @@ def delete_interface(self, interface): """ Delete interface from host - :param interface: Interface name - :type interface: str - :return: True/False - :rtype: bool + Args: + interface (str): Interface name + + Returns: + bool: True/false """ cmd = "ip link del %s" % interface try: @@ -593,10 +602,11 @@ def get_mac_by_ip(self, ip): """ Get mac address by ip address - :param ip: ip address - :type ip: str - :return: mac address - :rtype: str + Args: + ip (str): Ip address + + Returns: + str: Mac address """ interface = self.find_int_by_ip(ip=ip) return self.find_mac_by_int([interface])[0] @@ -605,10 +615,11 @@ def if_up(self, nic): """ Set nic up - :param nic: NIC name - :type nic: str - :return: True if setting NIC up succeeded, False otherwise - :rtype: bool + Args: + nic (str): Nic name + + Returns: + bool: True if setting nic up succeeded, false otherwise """ cmd = "ip link set up %s" % nic rc, _, _ = self.host.run_command(shlex.split(cmd)) @@ -618,10 +629,11 @@ def if_down(self, nic): """ Set nic down - :param nic: NIC name - :type nic: str - :return: True if setting NIC down succeeded, False otherwise - :rtype: bool + Args: + nic (str): Nic name + + Returns: + bool: True if setting nic down succeeded, false otherwise """ cmd = "ip link set down %s" % nic rc, _, _ = self.host.run_command(shlex.split(cmd)) @@ -631,11 +643,12 @@ def is_connective(self, ping_timeout=20.0): """ Check if host network is connective via ping command - :param ping_timeout: time to wait for response - :type ping_timeout: float - :return: True if address is connective via ping command, - False otherwise - :rtype: bool + Args: + ping_timeout (float): Time to wait for response + + Returns: + bool: True if address is connective via ping command, false + otherwise """ host_address = self.host.ip # Leave it for future support of IPV6 diff --git a/rrmngmnt/operatingsystem.py b/rrmngmnt/operatingsystem.py index a145410..dac2561 100644 --- a/rrmngmnt/operatingsystem.py +++ b/rrmngmnt/operatingsystem.py @@ -45,7 +45,8 @@ def get_release_info(self): It might raise exception in case the systemd is not deployed on system. - :raises: UnsupportedOperation + Raises: + UnsupportedOperation """ os_release_file = '/etc/os-release' cmd = ['cat', os_release_file] @@ -85,14 +86,16 @@ def get_distribution(self): """ Get OS info (Distro, version and code name) - :return: Results tuple(distname, version, id} - example: - Distribution( - distname='Red Hat Enterprise Linux Server', - id='Maipo', - version'='7.1' - ) - :rtype: namedtuple Distribution + Returns: + namedtuple Distribution: Results tuple(distname, version, id} + + Examples: + distribution( + distname='red hat enterprise linux server', + id='maipo', + version'='7.1' + ) + """ values = ["distname", "version", "id"] cmd = [ @@ -115,8 +118,8 @@ def stat(self, path): """ Get file or directory stats - :return: file stats - :rtype: collections.namedtuple + Returns: + collections.namedtuple: File stats """ type_map = { 'st_mode': ('0x%f', lambda x: int(x, 16)), @@ -158,8 +161,8 @@ def get_file_permissions(self, path): """ Get file permissions - :return: file permission in octal form(example 0644) - :rtype: str + Returns: + str: File permission in octal form(example 0644) """ cmd = ["stat", "-c", "%a", path] return self._exec_command(cmd=cmd).strip() @@ -168,8 +171,8 @@ def get_file_owner(self, path): """ Get file user and group owner name - :return: file user and group owner names(example ['root', 'root']) - :rtype: list + Returns: + list: File user and group owner names(example ['root', 'root']) """ cmd = ["stat", "-c", "%U %G", path] return self._exec_command(cmd=cmd).split() @@ -178,10 +181,11 @@ def user_exists(self, user_name): """ Check if user exist on system - :param user_name: user name - :type user_name: str - :return: True, if user exist, otherwise False - :rtype: bool + Args: + user_name (str): User name + + Returns: + bool: True, if user exist, otherwise false """ try: cmd = ["id", "-u", user_name] @@ -191,13 +195,15 @@ def user_exists(self, user_name): return True def group_exists(self, group_name): - """" + """ Check if group exist on system - :param group_name: group name - :type group_name: str - :return: True, if group exist, otherwise False - :rtype: bool + Args: + group_name (str): Group name + + Returns: + bool: True, if group exist, otherwise false + """ try: cmd = ["id", "-g", group_name] diff --git a/rrmngmnt/package_manager.py b/rrmngmnt/package_manager.py index 6d29f01..acdb61f 100644 --- a/rrmngmnt/package_manager.py +++ b/rrmngmnt/package_manager.py @@ -30,10 +30,11 @@ def _run_command_on_host(self, cmd): """ Run given command on host - :param cmd: command to run - :type cmd: list - :return: True, if command success, otherwise False - :rtype: bool + Args: + cmd (list): Command to run + + Returns: + bool: True, if command success, otherwise false """ self.logger.info( "Execute command '%s' on host %s", " ".join(cmd), self.host @@ -51,11 +52,14 @@ def exist(self, package): """ Check if package exist on host - :param package: name of package - :type package: str - :return: True, if package exist, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + package (str): Name of package + + Returns: + bool: True, if package exist, otherwise false + + Raises: + NotImplementedError """ if not self.exist_command_d: raise NotImplementedError("There is no 'exist' command defined.") @@ -68,11 +72,13 @@ def exist(self, package): def list_(self): """ - List installled packages on host + List installed packages on host - :return: installed packages - :rtype: list - :raise: NotImplementedError, CommandExecutionFailure + Returns: + list: Installed packages + + Raises: + NotImplementedError, CommandExecutionFailure """ if not self.list_command_d: raise NotImplementedError( @@ -97,11 +103,14 @@ def install(self, package): """ Install package on host - :param package: name of package - :type package: str - :return: True, if package installation success, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + package (str): Name of package + + Returns: + bool: True, if package installation success, otherwise false + + Raises: + NotImplementedError """ if not self.install_command_d: raise NotImplementedError("There is no 'install' command defined.") @@ -122,13 +131,16 @@ def remove(self, package, pattern=False): Remove package from host, or packages which match pattern if pattern is set to True - :param package: name of package or extended regular expression pattern - take a look at -E option in man grep - :type package: str - :param pattern: If True package name is pattern - :return: True, if package(s) removal success, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + pattern (bool): If true package name is pattern + package (str): Name of package or extended regular expression + pattern take a look at -e option in man grep + + Returns: + bool: True, if package(s) removal success, otherwise false + + Raises: + NotImplementedError """ if not self.remove_command_d: raise NotImplementedError("There is no 'remove' command defined.") @@ -174,11 +186,15 @@ def update(self, packages=None): if no packages are specified __author__ = "omachace" - :param packages: Packages to be updated, if empty, update system - :type packages: list - :return: True when updates succeed, False otherwise - :rtype: bool - :raise: NotImplementedError + + Args: + packages (list): Packages to be updated, if empty, update system + + Returns: + bool: True when updates succeed, false otherwise + + Raises: + NotImplementedError """ if not self.update_command_d: raise NotImplementedError("There is no 'update' command defined.") diff --git a/rrmngmnt/power_manager.py b/rrmngmnt/power_manager.py index a7d68df..bfb1463 100644 --- a/rrmngmnt/power_manager.py +++ b/rrmngmnt/power_manager.py @@ -99,12 +99,10 @@ def __init__(self, h, pm_if_type, pm_address, user): """ Initialize IPMIPowerManagement instance - :param pm_if_type: ipmi interface type(lan, lanplus) - :type pm_if_type: str - :param pm_address: power management address - :type pm_address: str - :param user: instance of User with pm username and password - :type user: User + Args: + pm_if_type (str): Ipmi interface type(lan, lanplus) + pm_address (str): Power management address + user (User): Instance of user with pm username and password """ super(IPMIPowerManager, self).__init__(h) self.pm_if_type = pm_if_type diff --git a/rrmngmnt/service.py b/rrmngmnt/service.py index edb1f95..35ead2c 100644 --- a/rrmngmnt/service.py +++ b/rrmngmnt/service.py @@ -83,7 +83,8 @@ def unmask(self): def _can_handle(self): """ - :raises: CanNotHandle + Raises: + CanNotHandle """ executor = self.host.executor() rc, _, _ = executor.run_cmd( diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 1743111..78ab521 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -202,12 +202,10 @@ def run(self, input_, timeout=None, get_pty=False): def __init__(self, user, address, use_pkey=False): """ - :param user: user - :type user: instance of User - :param address: ip / hostname - :type address: str - :param use_pkey: use ssh private key in the connection - :type use_pkey: bool + Args: + use_pkey (bool): Use ssh private key in the connection + user (instance of User): User + address (str): Ip / hostname """ super(RemoteExecutor, self).__init__(user) self.address = address @@ -215,25 +213,24 @@ def __init__(self, user, address, use_pkey=False): def session(self, timeout=None): """ - :param timeout: tcp timeout - :type timeout: float - :return: the session - :rtype: instance of RemoteExecutor.Session + Args: + timeout (float): Tcp timeout + + Returns: + instance of RemoteExecutor.Session: The session """ return RemoteExecutor.Session(self, timeout, self.use_pkey) def run_cmd(self, cmd, input_=None, tcp_timeout=None, io_timeout=None): """ - :param cmd: command - :type cmd: list - :param input_: input data - :type input_: str - :param tcp_timeout: tcp timeout - :type tcp_timeout: float - :param io_timeout: timeout for data operation (read/write) - :type io_timeout: float - :return: rc, out, err - :rtype: tuple (int, str, str) + Args: + tcp_timeout (float): Tcp timeout + cmd (list): Command + input_ (str): Input data + io_timeout (float): Timeout for data operation (read/write) + + Returns: + tuple (int, str, str): Rc, out, err """ with self.session(tcp_timeout) as session: return session.run_cmd(cmd, input_, io_timeout) @@ -242,10 +239,11 @@ def is_connective(self, tcp_timeout=20.0): """ Check if address is connective via ssh - :param tcp_timeout: time to wait for response - :type tcp_timeout: float - :return: True if address is connective, False otherwise - :rtype: bool + Args: + tcp_timeout (float): Time to wait for response + + Returns: + bool: True if address is connective, false otherwise """ try: self.logger.info( @@ -268,15 +266,14 @@ def wait_for_connectivity_state( """ Wait until address will be connective or not via ssh - :param positive: wait for the positive or negative connective state - :type positive: bool - :param timeout: wait timeout - :type timeout: int - :param sample_time: sample the ssh each sample_time seconds - :type sample_time: int - :return: True, if positive and ssh is connective or - negative and ssh does not connective, otherwise False - :rtype: bool + Args: + positive (bool): Wait for the positive or negative connective state + timeout (int): Wait timeout + sample_time (int): Sample the ssh each sample_time seconds + + Returns: + bool: True, if positive and ssh is connective or negative and ssh + does not connective, otherwise false """ reachable = "unreachable" if positive else "reachable" timeout_counter = 0 diff --git a/rrmngmnt/storage.py b/rrmngmnt/storage.py index 179a010..2385dd8 100644 --- a/rrmngmnt/storage.py +++ b/rrmngmnt/storage.py @@ -17,16 +17,15 @@ def mount(self, source, target=None, opts=None): Mounts source to target mount point __author__ = "ratamir" - :param source: Full path to source - :type source: str - :param target: Path to target directory, if omitted, a temporary - folder is created instead - :type target: str - :param opts: List of mount options such as: - ['-t', 'nfs', '-o', 'vers=3'] - :type opts: list - :return: Path to mount point if succeeded, None otherwise - :rtype: str + + Args: + source (str): Full path to source + target (str): Path to target directory, if omitted, a temporary + folder is created instead + opts (list): List of mount options such as + + Returns: + str: Path to mount point if succeeded, none otherwise """ target = '/tmp/mnt_point' if target is None else target cmd = ['mkdir', '-p', target] @@ -59,17 +58,17 @@ def umount(self, mount_point, force=True, remove_mount_point=True): optionally removes 'mount_point' __author__ = "ratamir" - :param mount_point: Path to directory that should be unmounted - :type mount_point: str - :param force: True if the mount point should be forcefully removed - (such as in the case of an unreachable NFS server) - :type force: bool - :param remove_mount_point: True if mount point should be deleted - after 'umount' operation completes, False otherwise - :type remove_mount_point: bool - :return: True if umount operation and mount point removal - succeeded, False otherwise - :rtype: bool + + Args: + mount_point (str): Path to directory that should be unmounted + force (bool): True if the mount point should be forcefully removed + (such as in the case of an unreachable nfs server) + remove_mount_point (bool): True if mount point should be deleted + after 'umount' operation completes, false otherwise + + Returns: + bool: True if umount operation and mount point removal succeeded, + false otherwise """ cmd = ['umount', mount_point, '-v'] if force: @@ -99,17 +98,18 @@ def lvchange(self, vg_name, lv_name, activate=True): (by setting it's 'active' attribute) __author__ = "ratamir" - :param vg_name: The name of the Volume group under which the LV resides - :type vg_name: str - :param lv_name: The name of the logical volume which will be - activated or deactivated - :type lv_name: str - :param activate: True when the logical volume should be activated, - False when it should be deactivated - :type activate: bool - :returns: True if setting the logical volume 'active' flag - succeeded, False otherwise - :rtype: bool + + Args: + activate (bool): True when the logical volume should be activated, + false when it should be deactivated + vg_name (str): The name of the volume group under which the lv + resides + lv_name (str): The name of the logical volume which will be + activated or deactivated + + Returns: + bool: True if setting the logical volume 'active' flag succeeded, + false otherwise """ active = 'y' if activate else 'n' return self.host.run_command( @@ -121,7 +121,8 @@ def pvscan(self): Execute 'pvscan' in order to get the current list of physical volumes __author__ = "ratamir" - :returns: True if the pvscan command succeded, False otherwise - :rtype: bool + + Returns: + bool: True if the pvscan command succeded, false otherwise """ return self.host.run_command(['pvscan'])[0] == 0 diff --git a/rrmngmnt/user.py b/rrmngmnt/user.py index c9737c1..ff590c8 100644 --- a/rrmngmnt/user.py +++ b/rrmngmnt/user.py @@ -4,10 +4,9 @@ class User(Resource): def __init__(self, name, password): """ - :param name: user name - :type name: str - :param password: password - :type password: str + Args: + password (str): Password + name (str): User name """ super(User, self).__init__() self.name = name @@ -31,12 +30,10 @@ def __init__(self, password): class Domain(Resource): def __init__(self, name, provider=None, server=None): """ - :param name: name of domain - :type name: str - :param provider: name of provider / type of domain - :type provider: str - :param server: server address - :type server: str + Args: + server (str): Server address + name (str): Name of domain + provider (str): Name of provider / type of domain """ super(Domain, self).__init__() self.name = name @@ -54,12 +51,10 @@ def __init__(self): class ADUser(User): def __init__(self, name, password, domain): """ - :param name: user name - :type name: str - :param password: password - :type password: str - :param domain: user domain - :type domain: instance of Domain + Args: + domain (instance of Domain): User domain + password (str): Password + name (str): User name """ super(ADUser, self).__init__(name, password) self.domain = domain From 3311a27d055c937201947c72cb82e7061ac1488c Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 3 May 2017 14:16:20 +0200 Subject: [PATCH 24/55] Added py36 matrix and classifier (#87) --- .travis.yml | 1 + setup.cfg | 1 + tox.ini | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 949bcd9..d72e3b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.7" - "3.4" - "3.5" + - "3.6" install: - pip install tox tox-travis codecov script: diff --git a/setup.cfg b/setup.cfg index 3987576..7b9940a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 keywords = remote resource diff --git a/tox.ini b/tox.ini index 572314a..d54d9ca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ [tox] -envlist = py27,py34,py35,pep8 +envlist = py27,py34,py35,py36,pep8 [tox:travis] 2.7 = py27, pep8 3.4 = py34, pep8 3.5 = py35, pep8 +3.6 = py36, pep8 [testenv] deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt From afb95f9f2f3f76bfd9457e60117473341846d1e1 Mon Sep 17 00:00:00 2001 From: Alexey Slaykovsky Date: Mon, 15 May 2017 14:07:21 +0200 Subject: [PATCH 25/55] [fs] Added create file and isexec methods (#88) Signed-off-by: Aleksei Slaikovskii --- rrmngmnt/filesystem.py | 16 ++++++++++++++++ tests/test_filesystem.py | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index e46734d..85a97c4 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -36,6 +36,9 @@ def isfile(self, path): def isdir(self, path): return self._exec_file_test('d', path) + def isexec(self, path): + return self._exec_file_test('x', path) + def remove(self, path): return self.host.executor().run_cmd( ['rm', '-f', path] @@ -131,6 +134,19 @@ def move(self, source_path, destination_path): cmd = ["mv", source_path, destination_path] return self.host.run_command(cmd)[0] == 0 + def create_file(self, content, path): + """ + Create file with given content on filesystem. + + Args: + content (str): content of the file. + path (str): destination path of the file. + """ + executor = self.host.executor() + with executor.session() as session: + with session.open_file(path, 'wb') as fh: + fh.write(six.b(content)) + def create_script(self, content, path): """ Create script on filesystem, and make it executable. diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index b41d20c..34085f3 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -33,6 +33,8 @@ class TestFilesystem(object): '[ -d /path/to/file1 ]': (1, '', ''), '[ -d /path/to ]': (0, '', ''), '[ -d somefile ]': (1, '', ''), + '[ -x /tmp/executable ]': (0, '', ''), + '[ -x /tmp/nonexecutable ]': (1, '', ''), 'rm -f /path/to/remove': (0, '', ''), 'rm -f /dir/to/remove': ( 1, '', 'rm: cannot remove ‘.tox/’: Is a directory', @@ -90,6 +92,12 @@ def test_isdir_positive(self): def test_isdir_negative(self): assert not self.get_host().fs.isdir('/tmp/nodir') + def test_isexec_positive(self): + assert self.get_host().fs.isexec('/tmp/executable') + + def test_isexec_negative(self): + assert not self.get_host().fs.isexec('/tmp/nonexecutable') + def test_remove_positive(self): assert self.get_host().fs.remove('/path/to/remove') @@ -112,7 +120,13 @@ def test_move(self): def test_flush_file(self): assert self.get_host().fs.flush_file("/tmp/file_to_flush") - def test_create_sctript(self): + def test_create_file(self): + data = "hello world" + path = "/tmp/hello.txt" + self.get_host().fs.create_file(data, path) + assert self.files[path].data == data + + def test_create_script(self): data = "echo hello" path = '/tmp/hello.sh' self.get_host().fs.create_script(data, path) From 10ce16e9d9ab5b49b8b56af5a04e73e5f4d7dea2 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Mon, 5 Jun 2017 09:59:28 +0300 Subject: [PATCH 26/55] network: add tcp and IO timeout to if_down function (#89) --- rrmngmnt/network.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index f005dbf..5821789 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -625,18 +625,24 @@ def if_up(self, nic): rc, _, _ = self.host.run_command(shlex.split(cmd)) return not bool(rc) - def if_down(self, nic): + def if_down(self, nic, tcp_timeout=20, io_timeout=20): """ Set nic down Args: nic (str): Nic name + tcp_timeout (float): TCP timeout + io_timeout (float): Timeout for data operation (read/write) Returns: bool: True if setting nic down succeeded, false otherwise """ cmd = "ip link set down %s" % nic - rc, _, _ = self.host.run_command(shlex.split(cmd)) + rc, _, _ = self.host.run_command( + command=shlex.split(cmd), + tcp_timeout=tcp_timeout, + io_timeout=io_timeout + ) return not bool(rc) def is_connective(self, ping_timeout=20.0): From 9c492e15838083e2f23f6838fe69d2c6b1d22584 Mon Sep 17 00:00:00 2001 From: lzitnits Date: Mon, 17 Jul 2017 17:20:22 +0300 Subject: [PATCH 27/55] Fix formatting error in lvchange function (#91) --- rrmngmnt/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rrmngmnt/storage.py b/rrmngmnt/storage.py index 2385dd8..f76cf5d 100644 --- a/rrmngmnt/storage.py +++ b/rrmngmnt/storage.py @@ -113,7 +113,7 @@ def lvchange(self, vg_name, lv_name, activate=True): """ active = 'y' if activate else 'n' return self.host.run_command( - shlex.split(LV_CHANGE_CMD % active, vg_name, lv_name) + shlex.split(LV_CHANGE_CMD % (active, vg_name, lv_name)) )[0] == 0 def pvscan(self): From c470943268e6b78e466162fc642262e40830fd83 Mon Sep 17 00:00:00 2001 From: lzitnits Date: Mon, 21 Aug 2017 10:09:23 +0300 Subject: [PATCH 28/55] Add firewall module (#85) --- README.rst | 13 +++ rrmngmnt/firewall.py | 174 +++++++++++++++++++++++++++++++++++++++++ rrmngmnt/host.py | 5 ++ tests/test_firewall.py | 127 ++++++++++++++++++++++++++++++ 4 files changed, 319 insertions(+) create mode 100644 rrmngmnt/firewall.py create mode 100644 tests/test_firewall.py diff --git a/README.rst b/README.rst index 26d6c54..de3ecbb 100644 --- a/README.rst +++ b/README.rst @@ -71,6 +71,19 @@ You can also mount devices. mp.remount('rw,sync,guest') # remount with different options h.fs.touch('%s/new_file' % mp.target) # touch file +Firewall +~~~~~~~~ + +Allows to manage firewall configurarion. Check which firewall service is +running on host (firewalld/iptables) and make configure this service. + +.. code:: python + + h.firewall.is_active('iptables') + h.firewall.chain('OUTPUT').list_rules() + h.firewall.chain('OUTPUT').add_rule('1.1.1.1', 'DROP') + + Network ~~~~~~~ diff --git a/rrmngmnt/firewall.py b/rrmngmnt/firewall.py new file mode 100644 index 0000000..713c507 --- /dev/null +++ b/rrmngmnt/firewall.py @@ -0,0 +1,174 @@ +from rrmngmnt.service import Service + +IPTABLES = 'iptables' + + +class Firewall(Service): + """ + Class for firewall services + """ + def __init__(self, host): + """ + Args: + host (host): Host object to run commands on + """ + super(Firewall, self).__init__(host) + self.host = host + + def is_active(self, firewall_service): + """ + Check if the relevant firewall service is active on the host + + Args: + firewall_service (str): Service name + + Returns: + bool: True if the service is active on host, False if not + + """ + return self.host.service(firewall_service).status() + + def chain(self, chain_name): + """ + Return Chain class to run commands on specefic firewall chain + Args: + chain_name (str): Name of chain to make changes + + Returns: + chain: Chain class object + + """ + return Chain(self.host, chain_name) + + +class Chain(Service): + """ + Class for Firewall specific chain commands + """ + def __init__(self, host, chain_name): + """ + Args: + host (host): Host object to run commands on + chain_name (str): Name of the firewall chain + """ + super(Chain, self).__init__(host) + self.host = host + self.firewall_service = IPTABLES + self.chain_name = chain_name.upper() + if self.chain_name == 'OUTPUT': + self.address_type = '--destination' + elif self.chain_name == 'INPUT': + self.address_type = '--source' + else: + raise NotImplementedError("only INPUT/OUTPUT chains are supported") + + def edit_chain( + self, action, chain_name, address_type, dest, target, protocol='all', + ports=None + ): + """ + Changes firewall configuration + + Args: + action (str): action to perform + chain_name (str): affected chain name + address_type (str): '--destination' for outgoing rules, + '--source' for incoming + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + + Returns: + bool: True if configuration change succeeded, False otherwise + + Raises: + NotImplementedError: In case the users specifies more than 15 ports + to block + + Example: + edit_chain( + action='--append',chain='OUTPUT', address_type='--destination', + dest={'address': nfs_server}, target='DROP' + ) + """ + dest = ",".join(dest['address']) + cmd = [ + self.firewall_service, action, chain_name, address_type, dest, + '--jump', target.upper(), '--protocol', protocol + ] + + if ports: + # Iptables multiport module accepts up to 15 ports + if len(ports) > 15: + raise NotImplementedError("Up to 15 ports can be specified") + ports = ",".join(ports) + + if protocol.lower() == 'all': + # Adjust the protocol type, '--dports' option requires specific + # type + cmd[-1] = 'tcp' + + cmd.extend(['--match', 'multiport', '--dports', ports]) + + return not self.host.executor().run_cmd(cmd)[0] + + def list_rules(self): + """ + List all existing rules in a specific Chain + + Returns: + list: List of existing rules + """ + cmd = [self.firewall_service, '--list-rules', self.chain_name] + rules = self.host.executor().run_cmd(cmd)[1] + return rules.splitlines() + + def add_rule(self, dest, target, protocol='all', ports=None): + """ + Add new firewall rule to a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + + Returns: + bool: False if adding new rule failed, True if it succeeded + """ + return self.edit_chain( + '--append', self.chain_name, self.address_type, dest, target, + protocol, ports + ) + + def delete_rule(self, dest, target, protocol='all', ports=None): + """ + Delete existing firewall rule from a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + + Returns: + bool: False if deleting rule failed, True if it succeeded + """ + return self.edit_chain( + '--delete', self.chain_name, self.address_type, dest, target, + protocol, ports + ) + + def clean_rules(self): + """ + Delete all rules in a specific chain + + Returns: + bool: True if succeeded, False otherwise + """ + cmd = [self.firewall_service, '--flush', self.chain_name] + return not self.host.executor().run_cmd(cmd)[0] diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 4d58613..f8dbc6e 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -15,6 +15,7 @@ from rrmngmnt import ssh from rrmngmnt.common import fqdn2ip from rrmngmnt.filesystem import FileSystem +from rrmngmnt.firewall import Firewall from rrmngmnt.network import Network from rrmngmnt.operatingsystem import OperatingSystem from rrmngmnt.package_manager import PackageManagerProxy @@ -499,3 +500,7 @@ def is_connective(self, tcp_timeout=20.0): "Use Host.executor().is_connective() instead." ) return self.executor().is_connective(tcp_timeout=tcp_timeout) + + @property + def firewall(self): + return Firewall(self) diff --git a/tests/test_firewall.py b/tests/test_firewall.py new file mode 100644 index 0000000..b5e7589 --- /dev/null +++ b/tests/test_firewall.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +import pytest +from rrmngmnt import Host +from rrmngmnt.user import RootUser +from .common import FakeExecutor + +host_executor = Host.executor + + +def teardown_module(): + Host.executor = host_executor + + +def fake_cmd_data(cmd_to_data): + def executor(self, user=None, pkey=False): + e = FakeExecutor(user, self.ip) + e.cmd_to_data = cmd_to_data.copy() + return e + Host.executor = executor + + +def get_host(ip='1.1.1.1'): + h = Host(ip) + h.users.append(RootUser('123456')) + return h + + +class TestFirewall(object): + + data = { + 'which systemctl': (0, '/usr/bin/systemctl', ''), + 'systemctl list-unit-files | grep -o ^[^.][^.]*.service ' + '| cut -d. -f1 | sort | uniq': ( + 0, + '\n'.join( + [ + 'iptables', + 'noniptables', + ] + ), + '' + ), + 'systemctl status iptables.service': (0, '', ''), + 'systemctl status noniptables.service': (1, '', ''), + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data) + + def test_running_service_positive(self): + assert get_host().firewall.is_active('iptables') + + +class TestChain(object): + + data = { + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --append INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --delete OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --delete INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol tcp --match multiport --dports ' + '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15': (0, '', ''), + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol tcp --match multiport --dports ' + '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, 16': ( + 4, '', 'iptables v1.4.21: too many ports specified' + ), + 'iptables --flush OUTPUT': (0, '', '') + } + + destination_host = {'address': ['2.2.2.2']} + ports = [ + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', + '14', '15' + ] + too_many_ports = [ + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', + '14', '15', '16' + ] + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data) + + def test_wrong_chain_name(self): + with pytest.raises(NotImplementedError): + get_host().firewall.chain('CHAIN') + + def test_add_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP' + ) + + def test_add_incoming_rule(self): + assert get_host().firewall.chain('INPUT').add_rule( + self.destination_host, 'DROP' + ) + + def test_delete_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').delete_rule( + self.destination_host, 'DROP' + ) + + def test_delete_incoming_rule(self): + assert get_host().firewall.chain('OUTPUT').delete_rule( + self.destination_host, 'DROP' + ) + + def test_add_outgoing_rule_with_ports(self): + assert get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP', ports=self.ports + ) + + def test_add_outgoing_rule_with_too_many_ports(self): + with pytest.raises(NotImplementedError): + get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP', ports=self.too_many_ports + ) + + def test_clean_firewall_rules(self): + assert get_host().firewall.chain('OUTPUT').clean_rules() From 9d557c76e07f057121d4113192113f0466a10fdb Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Thu, 23 Nov 2017 11:53:33 +0200 Subject: [PATCH 29/55] Network: Add netmask to network.get_info (#93) * Network: Add netmask to network.get_info * Network: Add netmask to network.get_info --- rrmngmnt/network.py | 4 ++++ tests/test_network.py | 1 + 2 files changed, 5 insertions(+) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 5821789..5c25664 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -482,6 +482,10 @@ def get_info(self): ip = self.find_ip_by_default_gw(gateway, ips_and_mask) net_info["ip"] = ip if ip is not None: + mask = [ + mask.split("/")[-1] for mask in ips_and_mask if ip in mask + ] + net_info["prefix"] = mask[0] if mask else "N/A" interface = self.find_int_by_ip(ip) # strip @NONE for PPC try: diff --git a/tests/test_network.py b/tests/test_network.py index 16de8ae..06e330f 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -205,6 +205,7 @@ def test_get_info(self): 'ip': '10.11.12.83', 'gateway': '10.11.12.254', 'interface': 'enp5s0f0', + 'prefix': '24' } assert info == expected_info From 397793f73004d27e77140e0be32b57b32f722885 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Mon, 27 Nov 2017 11:18:46 +0100 Subject: [PATCH 30/55] pypi deployment: update secure token --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d72e3b5..5f70d53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,7 @@ deploy: provider: pypi user: "lukas-bednar" password: - secure: "\ - ks4q6t0YBc4i3hr5uYCepUi05SuBfkA6l2vakuqcQunuwClaCN3ryP5aKCKk3673wdKBh2ee\ - L+VrKrmEnyRTrgo+t02ODSibAMeytwq254m526FiUbATemNrDyPtv7XTO/Yp9yFPwHbpoH8b\ - dTa4MhTUm6qXcRtRdYvfU8zVKUU=" + secure: "ks4q6t0YBc4i3hr5uYCepUi05SuBfkA6l2vakuqcQunuwClaCN3ryP5aKCKk3673wdKBh2eeL+VrKrmEnyRTrgo+t02ODSibAMeytwq254m526FiUbATemNrDyPtv7XTO/Yp9yFPwHbpoH8bdTa4MhTUm6qXcRtRdYvfU8zVKUU=" on: branch: master tags: true From af1074a02ae3ebde788ef3a4b95607136815d55d Mon Sep 17 00:00:00 2001 From: mkalfon Date: Mon, 15 Jan 2018 11:39:54 +0200 Subject: [PATCH 31/55] network.py module: add get interface speed function (#96) --- rrmngmnt/network.py | 19 +++++++++++++++++++ tests/test_network.py | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 5c25664..2a276bc 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -683,3 +683,22 @@ def is_connective(self, ping_timeout=20.0): ) return False return True + + def get_interface_speed(self, interface): + """ + Get network interface speed + + Args: + interface (str): Interface name + + Returns: + str: Interface speed, or empty string if error has occurred + """ + cmd = "ethtool -i {iface}".format(iface=interface) + rc, out, err = self.host.run_command(command=shlex.split(cmd)) + if rc: + logger.error("Error fetching speed from interface: %s", err) + return "" + return self._cmd( + ["cat", "/sys/class/net/{iface}/speed".format(iface=interface)] + ) diff --git a/tests/test_network.py b/tests/test_network.py index 06e330f..52d4ab4 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -190,6 +190,8 @@ class TestNetwork(object): ), 'ip link set up interface': True, 'ip link set down interface': True, + "ethtool -i eth0": (0, "driver: e1000", ""), + "cat /sys/class/net/eth0/speed": (0, "1000", "") } files = { } @@ -261,6 +263,9 @@ def if_up(self): def if_down(self): assert get_host().network.if_down("interface") + def test_get_interface_speed(self): + assert get_host().network.get_interface_speed("eth0") == "1000" + class TestHostNameCtl(object): From 32f356a05ac252a22dacfcbafc79f6a565b075ce Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Tue, 16 Jan 2018 11:38:33 +0200 Subject: [PATCH 32/55] network module raises Exception on command failure issue 51 (#97) Fixes #51 --- rrmngmnt/network.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 2a276bc..fc706a5 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -5,6 +5,7 @@ import shlex import six import subprocess +from rrmngmnt.errors import CommandExecutionFailure from rrmngmnt.service import Service @@ -127,7 +128,7 @@ def set_hostname(self, name): cmd = ['hostnamectl', 'set-hostname', name] rc, _, err = self._m.runCmd(cmd) if rc: - raise Exception("Unable to set hostname: %s" % err) + raise CommandExecutionFailure("Unable to set hostname: %s" % err) class Network(Service): @@ -142,7 +143,7 @@ def _cmd(self, cmd): if rc: cmd_out = " ".join(cmd) - raise Exception( + raise CommandExecutionFailure( "Fail to run command %s: %s ; %s" % (cmd_out, out, err)) return out From 75b37dadb20bf060dfb5cffd4951e86809f9a1c4 Mon Sep 17 00:00:00 2001 From: polinaag <35101331+polinaag@users.noreply.github.com> Date: Tue, 16 Jan 2018 13:45:19 +0200 Subject: [PATCH 33/55] Add possibility to insert iptables rule (#98) * Added possibility to give rule number for insert_rule method * added comments and fixed flake8 line length * fixed edit_chain brackets in a new line * fixed pep8 violations --- rrmngmnt/firewall.py | 49 ++++++++++++++++++++++++++++++++++++------ tests/test_firewall.py | 14 ++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/rrmngmnt/firewall.py b/rrmngmnt/firewall.py index 713c507..70589a6 100644 --- a/rrmngmnt/firewall.py +++ b/rrmngmnt/firewall.py @@ -64,7 +64,7 @@ def __init__(self, host, chain_name): def edit_chain( self, action, chain_name, address_type, dest, target, protocol='all', - ports=None + ports=None, rule_num=None ): """ Changes firewall configuration @@ -79,6 +79,8 @@ def edit_chain( target (str): target rule to apply protocol (str): affected network protocol, Default is 'all' ports (list): list of ports to configure + rule_num (str): the number given after the chain name indicates the + position where the rule will be inserted Returns: bool: True if configuration change succeeded, False otherwise @@ -89,16 +91,28 @@ def edit_chain( Example: edit_chain( - action='--append',chain='OUTPUT', address_type='--destination', - dest={'address': nfs_server}, target='DROP' + action='--append',chain='OUTPUT', + rule_num='1', + address_type='--destination', + dest={'address': nfs_server}, + target='DROP' ) """ - dest = ",".join(dest['address']) cmd = [ - self.firewall_service, action, chain_name, address_type, dest, - '--jump', target.upper(), '--protocol', protocol + self.firewall_service, action, chain_name ] + if rule_num: + cmd.extend([rule_num]) + + dest = ",".join(dest['address']) + cmd.extend( + [ + address_type, dest, '--jump', target.upper(), + '--protocol', protocol + ] + ) + if ports: # Iptables multiport module accepts up to 15 ports if len(ports) > 15: @@ -144,6 +158,29 @@ def add_rule(self, dest, target, protocol='all', ports=None): protocol, ports ) + def insert_rule(self, dest, target, protocol='all', ports=None, + rule_num=None): + """ + Insert new firewall rule to a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + rule_num (str): the number given after the chain name indicates + the position where the rule will be inserted. If the rule_num is + not given , the new rule is inserted in the line 1. + + Returns: + bool: False if inserting new rule failed, True if it succeeded + """ + return self.edit_chain( + '--insert', self.chain_name, self.address_type, dest, target, + protocol, ports, rule_num + ) + def delete_rule(self, dest, target, protocol='all', ports=None): """ Delete existing firewall rule from a specific chain diff --git a/tests/test_firewall.py b/tests/test_firewall.py index b5e7589..d979f3d 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -59,6 +59,10 @@ class TestChain(object): '--protocol all': (0, '', ''), 'iptables --append INPUT --source 2.2.2.2 --jump DROP ' '--protocol all': (0, '', ''), + 'iptables --insert OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --insert INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), 'iptables --delete OUTPUT --destination 2.2.2.2 --jump DROP ' '--protocol all': (0, '', ''), 'iptables --delete INPUT --source 2.2.2.2 --jump DROP ' @@ -102,6 +106,16 @@ def test_add_incoming_rule(self): self.destination_host, 'DROP' ) + def test_insert_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').insert_rule( + self.destination_host, 'DROP' + ) + + def test_insert_incoming_rule(self): + assert get_host().firewall.chain('INPUT').insert_rule( + self.destination_host, 'DROP' + ) + def test_delete_outgoing_rule(self): assert get_host().firewall.chain('OUTPUT').delete_rule( self.destination_host, 'DROP' From 3fad2c47c6c6ed23466b277b1133f77ce72deb85 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Tue, 16 Jan 2018 13:09:50 +0100 Subject: [PATCH 34/55] travis-deploy: updated password for pip --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f70d53..028f7d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,20 @@ language: python python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" +- '2.7' +- '3.4' +- '3.5' +- '3.6' install: - - pip install tox tox-travis codecov +- pip install tox tox-travis codecov script: - - tox +- tox after_success: - - codecov +- codecov deploy: provider: pypi - user: "lukas-bednar" + user: lukas-bednar password: - secure: "ks4q6t0YBc4i3hr5uYCepUi05SuBfkA6l2vakuqcQunuwClaCN3ryP5aKCKk3673wdKBh2eeL+VrKrmEnyRTrgo+t02ODSibAMeytwq254m526FiUbATemNrDyPtv7XTO/Yp9yFPwHbpoH8bdTa4MhTUm6qXcRtRdYvfU8zVKUU=" + secure: AlKh1Zv2+PjPa5W2RhHT0OtECmmNzDpSr+sWCDBFUnuZFiCLqXiD+aMxLCIbq0cndODhOd3MMXu3lKxNtmO5/7zUrSkHG0VRsqBj7yNabzJgp+rLX1FmKaw/20IuiNVhV31uh3MU1sHPBDNoU9S67SCiKk/L/k6t2wFTbac6gyiJ5D8HD4bukyhbGRf4GIFlibHLCq6GvSTCqEdSSnCVF/jSai4r5C7MicDpc3Osxud2EohnbcRT9aujKPxYzoTmworF2ZgIZqOUHoBo2oC8GdJlNOYdx/pBFYipMFxuWAHwZGiywxnob0ORgd54UxvTcvbXCIBPv4fjgrJRsJzoB2bH6QtG48RyCQIFrNOSDr/bPNP3XnvzG3yeAZ/Mxy50i9O4YhqgYum/VUBzvdU6SqFaDc/3FXt8dCz9AlPW1f2NMaElHREUJCXH17BjP1ziccUse7AhunHqMMLtIbuLAT8vZe0tJIAwgB/MSn9fFm4ME2oEeVRW4s8LVDIgc2LLKJ4AxwaZXfL6G8BgRV26EHgbPLwDMKDDKtaUL8KcY6TtTUx3pwDFrL+hOkhfE0Q302t6GI0UqG1t0pRweVfgpDHni4ReaHjDNXQqhKmMwrTRIQMVXPKoD3oHZMAyRtJW85e55aKVIO7iEEvhJCstU7tVmkCsJZcfFU//H+i9mVQ= on: branch: master tags: true From 57a131575fc54445af390867fe30c43afed77d54 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Mon, 16 Apr 2018 15:11:55 +0200 Subject: [PATCH 35/55] network: pass appropriate arguments to CommandExecutionFailure exception (#103) Signed-off-by: Lukas Bednar --- rrmngmnt/network.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index fc706a5..e8ae147 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -23,6 +23,10 @@ def __init__(self, host): self._s = None self._c = 0 + @property + def executor(self): + return self._e + def runCmd(self, cmd): return self._s.run_cmd(cmd) @@ -128,7 +132,8 @@ def set_hostname(self, name): cmd = ['hostnamectl', 'set-hostname', name] rc, _, err = self._m.runCmd(cmd) if rc: - raise CommandExecutionFailure("Unable to set hostname: %s" % err) + raise CommandExecutionFailure( + self._m.executor, cmd, rc, "Unable to set hostname: %s" % err) class Network(Service): @@ -142,9 +147,8 @@ def _cmd(self, cmd): rc, out, err = self._m.runCmd(cmd) if rc: - cmd_out = " ".join(cmd) raise CommandExecutionFailure( - "Fail to run command %s: %s ; %s" % (cmd_out, out, err)) + self._m.executor, cmd, rc, "OUT: %s\nERR: %s" % (out, err)) return out @keep_session From 45dabc08988272bd10cabf39e51fe7f84ea3740e Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Fri, 20 Apr 2018 14:28:51 +0200 Subject: [PATCH 36/55] tests: add docker runtime to run tests against to real system (#101) Signed-off-by: Lukas Bednar --- .travis.yml | 3 +++ rrmngmnt/executor.py | 5 +++++ rrmngmnt/host.py | 11 +++++++++- rrmngmnt/ssh.py | 25 ++++++++++++++++------ test-requirements.txt | 2 ++ tests/common.py | 22 ++++++++++---------- tests/docker-compose.yml | 7 +++++++ tests/test_db.py | 21 +++++++------------ tests/test_docker.py | 25 ++++++++++++++++++++++ tests/test_filesystem.py | 27 ++++++++++++------------ tests/test_firewall.py | 14 +++++-------- tests/test_network.py | 15 +++++--------- tests/test_os.py | 39 +++++++++++++++++++++-------------- tests/test_package_manager.py | 18 +++++++--------- tests/test_power_manager.py | 18 +++++++--------- tests/test_service.py | 20 ++++++++---------- 16 files changed, 162 insertions(+), 110 deletions(-) create mode 100644 tests/docker-compose.yml create mode 100644 tests/test_docker.py diff --git a/.travis.yml b/.travis.yml index 028f7d7..4152079 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,7 @@ language: python +sudo: required +services: + - docker python: - '2.7' - '3.4' diff --git a/rrmngmnt/executor.py b/rrmngmnt/executor.py index 3534ce7..3bfe09d 100644 --- a/rrmngmnt/executor.py +++ b/rrmngmnt/executor.py @@ -98,3 +98,8 @@ def run_cmd(self, cmd, input_=None): """ with self.session() as session: return session.run_cmd(cmd, input_) + + +class ExecutorFactory(object): + def build(self, host, user): + raise NotImplementedError() diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index f8dbc6e..917c4e9 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -39,6 +39,7 @@ class Host(Resource): SysVinit, InitCtl, ] + executor_factory = ssh.RemoteExecutorFactory() class LoggerAdapter(Resource.LoggerAdapter): """ @@ -219,7 +220,15 @@ def executor(self, user=None, pkey=False): """ if user is None: user = self.executor_user - return ssh.RemoteExecutor(user, self.ip, use_pkey=pkey) + if pkey: + warnings.warn( + "Parameter 'pkey' is deprecated and will be removed in future." + "Please use ssh.RemoteExecutorFactory to set this parameter." + ) + ef = copy.copy(ssh.RemoteExecutorFactory) + ef.use_pkey = pkey + return ef(self.ip, user) + return self.executor_factory.build(self, user) def run_command( self, command, input_=None, tcp_timeout=None, io_timeout=None, diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 78ab521..01db3ad 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -4,7 +4,7 @@ import paramiko import contextlib import subprocess -from rrmngmnt.executor import Executor +from rrmngmnt.executor import Executor, ExecutorFactory AUTHORIZED_KEYS = os.path.join("%s", ".ssh/authorized_keys") KNOWN_HOSTS = os.path.join("%s", ".ssh/known_hosts") @@ -49,14 +49,14 @@ class Session(Executor.Session): """ Represents active ssh connection """ - def __init__(self, executor, timeout=None, use_pkey=False): + def __init__(self, executor, timeout=None): super(RemoteExecutor.Session, self).__init__(executor) if timeout is None: timeout = RemoteExecutor.TCP_TIMEOUT self._timeout = timeout self._ssh = paramiko.SSHClient() self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - if use_pkey: + if self._executor.use_pkey: self.pkey = paramiko.RSAKey.from_private_key_file( ID_RSA_PRV % os.path.expanduser('~') ) @@ -85,7 +85,8 @@ def open(self): username=self._executor.user.name, password=self._executor.user.password, timeout=self._timeout, - pkey=self.pkey + pkey=self.pkey, + port=self._executor.port, ) except (socket.gaierror, socket.herror) as ex: args = list(ex.args) @@ -200,16 +201,18 @@ def run(self, input_, timeout=None, get_pty=False): self.err = err.read() return self.rc, self.out, self.err - def __init__(self, user, address, use_pkey=False): + def __init__(self, user, address, use_pkey=False, port=22): """ Args: use_pkey (bool): Use ssh private key in the connection user (instance of User): User address (str): Ip / hostname + port (int): Port to connect """ super(RemoteExecutor, self).__init__(user) self.address = address self.use_pkey = use_pkey + self.port = port def session(self, timeout=None): """ @@ -219,7 +222,7 @@ def session(self, timeout=None): Returns: instance of RemoteExecutor.Session: The session """ - return RemoteExecutor.Session(self, timeout, self.use_pkey) + return RemoteExecutor.Session(self, timeout) def run_cmd(self, cmd, input_=None, tcp_timeout=None, io_timeout=None): """ @@ -287,3 +290,13 @@ def wait_for_connectivity_state( time.sleep(sample_time) timeout_counter += sample_time return True + + +class RemoteExecutorFactory(ExecutorFactory): + def __init__(self, use_pkey=False, port=22): + self.use_pkey = use_pkey + self.port = port + + def build(self, host, user): + return RemoteExecutor( + user, host.ip, use_pkey=self.use_pkey, port=self.port) diff --git a/test-requirements.txt b/test-requirements.txt index 5b6c82e..b87e66c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,4 @@ pytest>=2.8.7 pytest-cov +pytest-docker +docker-compose diff --git a/tests/common.py b/tests/common.py index 04d47f9..11be733 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,6 @@ import contextlib from subprocess import list2cmdline -from rrmngmnt.executor import Executor +from rrmngmnt.executor import Executor, ExecutorFactory import six @@ -121,13 +121,13 @@ def run_cmd(self, cmd, input_=None, tcp_timeout=None, io_timeout=None): return session.run_cmd(cmd, input_, io_timeout) -if __name__ == "__main__": - from rrmngmnt import RootUser - u = RootUser('password') - e = FakeExecutor(u) - e.cmd_to_data = {'echo ahoj': (0, 'ahoj', '')} - print(e.run_cmd(['echo', 'ahoj'])) - with e.session() as ss: - with ss.open_file('/tmp/a', 'w') as fh: - fh.write("ahoj") - print(e.files_content['/tmp/a'], e.files_content['/tmp/a'].data) +class FakeExecutorFactory(ExecutorFactory): + def __init__(self, cmd_to_data, files_content): + self.cmd_to_data = cmd_to_data.copy() + self.files_content = files_content + + def build(self, host, user): + fe = FakeExecutor(user, host.ip) + fe.cmd_to_data = self.cmd_to_data.copy() + fe.files_content = self.files_content + return fe diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..3c5e189 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,7 @@ +--- +version: '3' +services: + ubuntu: + ports: + - "22221:22" + image: "chrismeyers/ubuntu12.04" diff --git a/tests/test_db.py b/tests/test_db.py index 4088cde..1670b1a 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -3,23 +3,18 @@ from rrmngmnt import Host, User from rrmngmnt.db import Database -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestDb(object): @@ -51,10 +46,10 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_db(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '34546')) return Database( - Host(ip), - self.db_name, - User(self.db_user, self.db_pass), + h, self.db_name, User(self.db_user, self.db_pass), ) def test_restart(self): diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 0000000..23bfe83 --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,25 @@ +import pytest +from rrmngmnt import Host, User +from rrmngmnt.ssh import RemoteExecutorFactory + + +@pytest.fixture(scope='session') +def provisioned_hosts(docker_ip, docker_services): + hosts = {} + for h in ('ubuntu',): + host = Host(docker_ip) + host.add_user(User("root", "docker.io")) + host.executor_factory = RemoteExecutorFactory( + port=docker_services.port_for(h, 22)) + executor = host.executor() + docker_services.wait_until_responsive( + timeout=30.0, pause=1, + check=lambda: executor.is_connective, + ) + hosts[h] = host + return hosts + + +def test_echo(provisioned_hosts): + ubuntu_host = provisioned_hosts['ubuntu'] + ubuntu_host.executor().run_cmd(['echo', 'hello']) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 34085f3..f859a06 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -3,23 +3,18 @@ from rrmngmnt import Host, User from rrmngmnt import errors -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=User('fakeuser', 'password'), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestFilesystem(object): @@ -72,7 +67,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_exists_positive(self): assert self.get_host().fs.exists('/tmp/exits') @@ -194,7 +191,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_get(self, tmpdir): self.get_host().fs.get("/path/to/get_file", str(tmpdir)) @@ -221,7 +220,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_transfer(self): self.get_host().fs.transfer( diff --git a/tests/test_firewall.py b/tests/test_firewall.py index d979f3d..98c655f 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -2,21 +2,17 @@ import pytest from rrmngmnt import Host from rrmngmnt.user import RootUser -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) def get_host(ip='1.1.1.1'): diff --git a/tests/test_network.py b/tests/test_network.py index 52d4ab4..7edae24 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -1,22 +1,17 @@ # -*- coding: utf-8 -*- from rrmngmnt import Host, RootUser -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) def get_host(ip='1.1.1.1'): diff --git a/tests/test_os.py b/tests/test_os.py index 26c4076..2c53713 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -3,23 +3,18 @@ from rrmngmnt import Host, User from rrmngmnt import errors -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=User("fake", "pass"), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestOperatingSystem(object): @@ -61,7 +56,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_get_release_str(self): result = self.get_host().os.release_str @@ -106,7 +103,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11114')) + return h def test_get_release_str(self): with pytest.raises(errors.CommandExecutionFailure) as ex_info: @@ -138,7 +137,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '22222')) + return h def test_get_release_info(self): with pytest.raises(errors.UnsupportedOperation) as ex_info: @@ -169,7 +170,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '3333')) + return h def test_get_release_info(self): info = self.get_host().os.release_info @@ -244,7 +247,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11331')) + return h def test_get_file_stats(self): file_stats = self.get_host().os.stat('/tmp/test') @@ -304,7 +309,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '155511')) + return h def test_get_file_stats(self): with pytest.raises(errors.CommandExecutionFailure) as ex_info: diff --git a/tests/test_package_manager.py b/tests/test_package_manager.py index 853d354..d968e04 100644 --- a/tests/test_package_manager.py +++ b/tests/test_package_manager.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from rrmngmnt import Host, User -from .common import FakeExecutor +from .common import FakeExecutorFactory import rrmngmnt.package_manager as pm from rrmngmnt.package_manager import PackageManagerProxy as PMProxy from subprocess import list2cmdline -host_executor = Host.executor +host_executor_factory = Host.executor_factory def extend_cmd(cmd, *args): @@ -22,15 +22,11 @@ def join_cmds(*args): def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=User('fakeuser', 'password'), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class BasePackageManager(object): @@ -130,7 +126,9 @@ def set_base_data(cls): }) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def get_pm(self): return self.get_host().package_manager diff --git a/tests/test_power_manager.py b/tests/test_power_manager.py index cd83cc2..7b0fcf3 100644 --- a/tests/test_power_manager.py +++ b/tests/test_power_manager.py @@ -1,7 +1,7 @@ import pytest from rrmngmnt import Host, User, power_manager -from .common import FakeExecutor +from .common import FakeExecutorFactory PM_TYPE = 'lanplus' PM_ADDRESS = 'test-mgmt.test' @@ -16,19 +16,15 @@ pm_password=PM_PASSWORD ) -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestPowerManager(object): @@ -50,7 +46,9 @@ def setup_class(cls): @staticmethod def get_host(ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '123456')) + return h class TestSSHPowerManager(TestPowerManager): diff --git a/tests/test_service.py b/tests/test_service.py index 0218f8a..3416661 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,27 +1,25 @@ # -*- coding: utf-8 -*- -from rrmngmnt import Host +from rrmngmnt import Host, User from rrmngmnt.service import SysVinit, Systemd, InitCtl -from .common import FakeExecutor +from .common import FakeExecutorFactory import pytest -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) def get_host(ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', 'fakepasswd')) + return h class TestSystemService(object): From 6baa74013e63c928fa4ef2ab54f1376cfdf40ca3 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Fri, 20 Apr 2018 14:37:37 +0200 Subject: [PATCH 37/55] ssh: translate bytes to string (#102) Signed-off-by: Lukas Bednar --- rrmngmnt/ssh.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 01db3ad..7979a79 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -5,6 +5,8 @@ import contextlib import subprocess from rrmngmnt.executor import Executor, ExecutorFactory +import six + AUTHORIZED_KEYS = os.path.join("%s", ".ssh/authorized_keys") KNOWN_HOSTS = os.path.join("%s", ".ssh/known_hosts") @@ -198,7 +200,11 @@ def run(self, input_, timeout=None, get_pty=False): in_.write(input_) in_.close() self.out = out.read() + if isinstance(self.out, six.binary_type): + self.out = self.out.decode('utf-8', errors='replace') self.err = err.read() + if isinstance(self.err, six.binary_type): + self.err = self.err.decode('utf-8', errors='replace') return self.rc, self.out, self.err def __init__(self, user, address, use_pkey=False, port=22): From 97dc5ee6f81cdae7ccc96eeb887fdc3a8814ae7b Mon Sep 17 00:00:00 2001 From: pmatyas Date: Tue, 15 May 2018 13:27:52 +0200 Subject: [PATCH 38/55] Add logger info with journalctl output when systemd service action fails --- rrmngmnt/service.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/service.py b/rrmngmnt/service.py index 35ead2c..b417d02 100644 --- a/rrmngmnt/service.py +++ b/rrmngmnt/service.py @@ -199,7 +199,14 @@ def _execute(self, action): ] executor = self.host.executor() rc, _, _ = executor.run_cmd(cmd, io_timeout=self.timeout) - return rc == 0 + + rc_result = rc == 0 + if not rc_result: + cmd = ['journalctl', '-u', self.name] + _, out, _ = executor.run_cmd(cmd, io_timeout=self.timeout) + self.logger.info(out) + + return rc_result def is_enabled(self): return self._execute('is-enabled') From c6bd126766cc71c5311de933c4edb6d8dd6d7da9 Mon Sep 17 00:00:00 2001 From: pmatyas Date: Tue, 15 May 2018 15:48:54 +0200 Subject: [PATCH 39/55] Fix systemd services test for journalctl addition --- rrmngmnt/service.py | 9 ++++----- tests/test_service.py | 4 ++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/rrmngmnt/service.py b/rrmngmnt/service.py index b417d02..066f4ce 100644 --- a/rrmngmnt/service.py +++ b/rrmngmnt/service.py @@ -200,13 +200,12 @@ def _execute(self, action): executor = self.host.executor() rc, _, _ = executor.run_cmd(cmd, io_timeout=self.timeout) - rc_result = rc == 0 - if not rc_result: - cmd = ['journalctl', '-u', self.name] + if rc: + cmd = ['journalctl', '-u', self.name + ".service"] _, out, _ = executor.run_cmd(cmd, io_timeout=self.timeout) - self.logger.info(out) + self.logger.warning(out) - return rc_result + return rc == 0 def is_enabled(self): return self._execute('is-enabled') diff --git a/tests/test_service.py b/tests/test_service.py index 3416661..246bda7 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -103,13 +103,17 @@ class TestSystemd(TestSystemService): ), 'systemctl is-enabled s-enabled.service': (0, '', ''), 'systemctl is-enabled s-disabled.service': (1, '', ''), + 'journalctl -u s-disabled.service': (0, '-- No entries --', ''), 'systemctl enable s-disabled.service': (0, '', ''), 'systemctl disable s-enabled.service': (0, '', ''), 'systemctl status s-running.service': (0, '', ''), 'systemctl status s-stopped.service': (1, '', ''), + 'journalctl -u s-stopped.service': (0, '-- No entries --', ''), 'systemctl start s-stopped.service': (0, '', ''), 'systemctl start s-running.service': (1, '', ''), + 'journalctl -u s-running.service': (0, '-- No entries --', ''), 'systemctl stop s-stopped.service': (1, '', ''), + 'journalctl -u s-stopped.service': (0, '-- No entries --', ''), 'systemctl stop s-running.service': (0, '', ''), 'systemctl restart s-running.service': (0, '', ''), 'systemctl reload s-running.service': (0, '', ''), From dd95420d38a65b03a5c0104ff49d9aaabeb298b5 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 18 Jun 2018 20:44:43 +0300 Subject: [PATCH 40/55] Network: Add get_interface_status function --- rrmngmnt/network.py | 27 +++++++++++++++++++-------- tests/test_network.py | 6 +++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index e8ae147..7f42f1d 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -699,11 +699,22 @@ def get_interface_speed(self, interface): Returns: str: Interface speed, or empty string if error has occurred """ - cmd = "ethtool -i {iface}".format(iface=interface) - rc, out, err = self.host.run_command(command=shlex.split(cmd)) - if rc: - logger.error("Error fetching speed from interface: %s", err) - return "" - return self._cmd( - ["cat", "/sys/class/net/{iface}/speed".format(iface=interface)] - ) + ethtool_cmd = "ethtool -i {iface}".format(iface=interface) + self._cmd(shlex.split(ethtool_cmd)) + speed_cmd = "cat /sys/class/net/{iface}/speed".format(iface=interface) + out = self._cmd(shlex.split(speed_cmd)) + return out.strip() + + def get_interface_status(self, interface): + """ + Get interface status + + Args: + interface (str): Interface name + + Returns: + str: Interface status (up/down) + """ + cmd = "cat /sys/class/net/{iface}/operstate".format(iface=interface) + out = self._cmd(shlex.split(cmd)) + return out.strip() diff --git a/tests/test_network.py b/tests/test_network.py index 7edae24..4b94572 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -186,7 +186,8 @@ class TestNetwork(object): 'ip link set up interface': True, 'ip link set down interface': True, "ethtool -i eth0": (0, "driver: e1000", ""), - "cat /sys/class/net/eth0/speed": (0, "1000", "") + "cat /sys/class/net/eth0/speed": (0, "1000", ""), + "cat /sys/class/net/eth0/operstate": (0, "up", "") } files = { } @@ -261,6 +262,9 @@ def if_down(self): def test_get_interface_speed(self): assert get_host().network.get_interface_speed("eth0") == "1000" + def test_get_interface_status(self): + assert get_host().network.get_interface_status("eth0") == "up" + class TestHostNameCtl(object): From 56d94814197ebb39da397c3436c677f184821dc5 Mon Sep 17 00:00:00 2001 From: Lucie Leistnerova Date: Wed, 22 Aug 2018 15:17:35 +0200 Subject: [PATCH 41/55] unicode fix for ssh (#106) * unicode fix for ssh * unicode fix - refactoring --- rrmngmnt/common.py | 17 +++++++++++++++++ rrmngmnt/ssh.py | 10 +++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/rrmngmnt/common.py b/rrmngmnt/common.py index 3e56e68..923e84b 100644 --- a/rrmngmnt/common.py +++ b/rrmngmnt/common.py @@ -1,3 +1,4 @@ +import six import socket @@ -20,3 +21,19 @@ def fqdn2ip(fqdn): ex.strerror = message ex.args = tuple(args) raise + + +def normalize_string(data): + """ + get normalized string + + Args: + data (object): data to process + Returns: + object: normalized string + """ + if isinstance(data, six.binary_type): + data = data.decode('utf-8', errors='replace') + if isinstance(data, six.text_type): + data = data.encode('utf-8', errors='replace') + return data diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 7979a79..4ce2a23 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -4,8 +4,8 @@ import paramiko import contextlib import subprocess +from rrmngmnt.common import normalize_string from rrmngmnt.executor import Executor, ExecutorFactory -import six AUTHORIZED_KEYS = os.path.join("%s", ".ssh/authorized_keys") @@ -199,12 +199,8 @@ def run(self, input_, timeout=None, get_pty=False): if input_: in_.write(input_) in_.close() - self.out = out.read() - if isinstance(self.out, six.binary_type): - self.out = self.out.decode('utf-8', errors='replace') - self.err = err.read() - if isinstance(self.err, six.binary_type): - self.err = self.err.decode('utf-8', errors='replace') + self.out = normalize_string(out.read()) + self.err = normalize_string(err.read()) return self.rc, self.out, self.err def __init__(self, user, address, use_pkey=False, port=22): From 74032b6443e26d2d8a20de4da82110daff30db14 Mon Sep 17 00:00:00 2001 From: MosheSheena Date: Tue, 30 Oct 2018 09:40:35 +0200 Subject: [PATCH 42/55] Fix some PEP comments that travis found (#109) There was one error of type: 'W605 invalid escape sequence' that was repeating itself in several places in the network and test_network modules --- rrmngmnt/network.py | 4 ++-- tests/test_network.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 7f42f1d..6fc8c07 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -190,7 +190,7 @@ def all_interfaces(self): list of strings: List of interfaces """ out = self._cmd( - "ls -la /sys/class/net | grep 'dummy_\|pci' | grep -o '[" + "ls -la /sys/class/net | grep 'dummy_\\|pci' | grep -o '[" "^/]*$'".split() ) out = out.strip().splitlines() @@ -398,7 +398,7 @@ def list_bridges(self): 'brctl', 'show', '|', 'sed', '-e', '/^bridge name/ d', # remove header # deal with multiple interfaces - '-e', "'s/^\s\s*\(\S\S*\)$/CONT:\\1/I'" + '-e', "'s/^\\s\\s*\\(\\S\\S*\\)$/CONT:\\1/I'" ] out = self._cmd(cmd).strip() if not out: diff --git a/tests/test_network.py b/tests/test_network.py index 4b94572..3e0b788 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -166,7 +166,7 @@ class TestNetwork(object): ), 'brctl addbr br1': (0, '', ''), 'brctl addif br1 net1': (0, '', ''), - 'ls -la /sys/class/net | grep \'dummy_\|pci\' | grep -o \'[^/]*$\'': ( + 'ls -la /sys/class/net | grep \'dummy_\\|pci\' | grep -o \'[^/]*$\'': ( 0, '\n'.join( [ From f577d018069932208aa5204dd78f0b9859fed54c Mon Sep 17 00:00:00 2001 From: MosheSheena Date: Wed, 31 Oct 2018 17:28:55 +0200 Subject: [PATCH 43/55] Change send_icmp to use OS default ping size (#110) * Change send_icmp to use OS default ping size Changed send_icmp function to use default host OS ping size in case a size is not passed to the fuction * Add test for send_icmp method Signed-off-by: Lukas Bednar --- rrmngmnt/network.py | 10 +++++----- tests/test_network.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 6fc8c07..9eb93f7 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -545,7 +545,7 @@ def delete_ifcfg_file(self, nic, ifcfg_path=IFCFG_PATH): return False return True - def send_icmp(self, dst, count="5", size="1500", extra_args=None): + def send_icmp(self, dst, count="5", size=None, extra_args=None): """ Send ICMP to destination IP/FQDN @@ -558,10 +558,10 @@ def send_icmp(self, dst, count="5", size="1500", extra_args=None): Returns: bool: True/false """ - cmd = ["ping", dst, "-c", count, "-s", size] - if size != "1500": - cmd.extend(["-M", "do"]) - if extra_args is not None: + cmd = ["ping", dst, "-c", count] + if size: + cmd.extend(["-s", size, "-M", "do"]) + if extra_args: for ar in extra_args.split(): cmd.extend(ar.split()) try: diff --git a/tests/test_network.py b/tests/test_network.py index 3e0b788..d2c2ba2 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -187,7 +187,8 @@ class TestNetwork(object): 'ip link set down interface': True, "ethtool -i eth0": (0, "driver: e1000", ""), "cat /sys/class/net/eth0/speed": (0, "1000", ""), - "cat /sys/class/net/eth0/operstate": (0, "up", "") + "cat /sys/class/net/eth0/operstate": (0, "up", ""), + "ping 1.2.3.4 -c 5 -s 10 -M do": (0, "something", ""), } files = { } @@ -265,6 +266,9 @@ def test_get_interface_speed(self): def test_get_interface_status(self): assert get_host().network.get_interface_status("eth0") == "up" + def test_send_icmp(self): + assert get_host().network.send_icmp('1.2.3.4', size="10") + class TestHostNameCtl(object): From 8ff1a192cad31deab50508a59d17e9ea1a9fc211 Mon Sep 17 00:00:00 2001 From: Lucie Leistnerova Date: Mon, 10 Dec 2018 09:06:05 +0100 Subject: [PATCH 44/55] Add -p, -m to fs.mkdir (#111) * unicode fix for ssh * unicode fix - refactoring * Add -p, -m to fs.mkdir * Added test for fs.mkdir -p,-m --- rrmngmnt/filesystem.py | 14 ++++++++++++-- tests/test_filesystem.py | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 85a97c4..64a928b 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -161,17 +161,27 @@ def create_script(self, content, path): fh.write(six.b(content)) self.chmod(path=path, mode="+x") - def mkdir(self, path): + def mkdir(self, path, parents=False, mode=None): """ Create directory on host Args: path (str): directory path + parents (bool): True - no error if existing, make parent + directories as needed, False - error when parent + doesn't exist (default False) + mode (str): permission mode(600 for example or u+x) Raises: CommandExecutionFailure: If mkdir failed """ - self._exec_command(['mkdir', path]) + cmd = ['mkdir'] + if parents: + cmd.append('-p') + if mode: + cmd.extend(['-m', mode]) + cmd.append(path) + self._exec_command(cmd) def chown(self, path, username, groupname): """ diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index f859a06..441c88a 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -38,6 +38,7 @@ class TestFilesystem(object): 'cat %s' % "/tmp/file": (0, 'data', ''), 'chmod +x /tmp/hello.sh': (0, '', ''), 'mkdir /dir/to/remove': (0, '', ''), + 'mkdir -p -m 600 /dir/to/remove2/remove': (0, '', ''), 'chown root:root /dir/to/remove': (0, '', ''), 'chmod 600 /dir/to/remove': (0, '', ''), 'chmod 600 /tmp/nofile': ( @@ -132,6 +133,11 @@ def test_create_script(self): def test_mkdir_positive(self): self.get_host().fs.mkdir('/dir/to/remove') + def test_mkdir_pm_positive(self): + self.get_host().fs.mkdir( + '/dir/to/remove2/remove', parents=True, mode='600' + ) + def test_chown_positive(self): self.get_host().fs.chown('/dir/to/remove', 'root', 'root') From 518c3c4ea856111f5050067346c82348cb43a3f5 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Fri, 8 Feb 2019 12:13:56 +0100 Subject: [PATCH 45/55] Add Database.psql_cmd to run special psql commands like \dt, \dv --- rrmngmnt/db.py | 27 ++++++++++++++++++++++++++- tests/test_db.py | 5 +++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/rrmngmnt/db.py b/rrmngmnt/db.py index d7ff163..b58b9ad 100644 --- a/rrmngmnt/db.py +++ b/rrmngmnt/db.py @@ -16,7 +16,7 @@ def __init__(self, host, name, user): def psql(self, sql, *args): """ - Execute psql command on host + Execute sql command on host Args: sql (str): sql command @@ -48,6 +48,31 @@ def psql(self, sql, *args): # I need to think whether it is better or not. # We need to realize that connection can be forbidden from outside ... + def psql_cmd(self, command): + """ + Execute psql special command on host (e.g. \dt, \dv, ...) + + Args: + command (str): special psql command + Returns: + str: output of the command + """ + cmd = [ + 'export', 'PGPASSWORD=%s;' % self.user.password, + 'psql', '-d', self.name, '-U', self.user.name, '-h', 'localhost', + '-c', command + ] + executor = self.host.executor() + with executor.session() as ss: + rc, out, err = ss.run_cmd(cmd) + if rc: + raise Exception( + "Failed to exec command: %s" % err + ) + if not out and err: + out = err + return out + def restart(self): """ Restart postgresql service diff --git a/tests/test_db.py b/tests/test_db.py index 1670b1a..b45a142 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -66,3 +66,8 @@ def test_negative(self): with pytest.raises(Exception) as ex_info: db.psql("SELECT * FROM table ERROR") assert "Syntax Error" in str(ex_info.value) + + def test_psql_cmd(self): + db = self.get_db() + res = db.psql_cmd('\dt') + assert res From 5cd29dd555aecc71152b0152d2a557abdd26f4a2 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Fri, 8 Feb 2019 13:15:21 +0100 Subject: [PATCH 46/55] Fixed test for psql_cmd --- tests/test_db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_db.py b/tests/test_db.py index b45a142..3356160 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -34,6 +34,10 @@ class TestDb(object): '-R __RECORD_SEPARATOR__ -t -A -c "SELECT * FROM table ERROR"': ( 1, "", "Syntax Error" ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c "\\\dt"': ( + 0, "", "" + ), } files = {} @@ -69,5 +73,5 @@ def test_negative(self): def test_psql_cmd(self): db = self.get_db() - res = db.psql_cmd('\dt') + res = db.psql_cmd('\\\dt') assert res From a632e8167b18b5704678b0755c676277439fca77 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Fri, 8 Feb 2019 13:35:04 +0100 Subject: [PATCH 47/55] Fixed test for psql_cmd II. --- tests/test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_db.py b/tests/test_db.py index 3356160..2413f35 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -35,7 +35,7 @@ class TestDb(object): 1, "", "Syntax Error" ), 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' - '-c "\\\dt"': ( + '-c \\\dt': ( 0, "", "" ), } From 614fe73d9597462c8ef91ee5f3bda2f5eaee3d92 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Fri, 8 Feb 2019 15:32:46 +0100 Subject: [PATCH 48/55] Fixed test for psql_cmd III. --- tests/test_db.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_db.py b/tests/test_db.py index 2413f35..0329679 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -36,7 +36,18 @@ class TestDb(object): ), 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' '-c \\\dt': ( - 0, "", "" + 0, + ( + "List of relations\n" + " Schema | Name | Type | Owner\n" + "--------+----------------------+-------+---------\n" + " public | test_table | table | postgres\n" + ), + "" + ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c \\\dv': ( + 0, "", "Did not find any relations." ), } files = {} @@ -74,4 +85,6 @@ def test_negative(self): def test_psql_cmd(self): db = self.get_db() res = db.psql_cmd('\\\dt') - assert res + assert 'List of relations' in res + res = db.psql_cmd('\\\dv') + assert res == 'Did not find any relations.' From bdebedc39513b6dc13716c623dc2775b89b809a1 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Mon, 11 Feb 2019 08:33:14 +0100 Subject: [PATCH 49/55] Fixed commands for tests --- rrmngmnt/db.py | 2 +- tests/test_db.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rrmngmnt/db.py b/rrmngmnt/db.py index b58b9ad..5534353 100644 --- a/rrmngmnt/db.py +++ b/rrmngmnt/db.py @@ -50,7 +50,7 @@ def psql(self, sql, *args): def psql_cmd(self, command): """ - Execute psql special command on host (e.g. \dt, \dv, ...) + Execute psql special command on host (e.g. \\dt, \\dv, ...) Args: command (str): special psql command diff --git a/tests/test_db.py b/tests/test_db.py index 0329679..8174731 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -35,7 +35,7 @@ class TestDb(object): 1, "", "Syntax Error" ), 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' - '-c \\\dt': ( + '-c \\\\dt': ( 0, ( "List of relations\n" @@ -46,7 +46,7 @@ class TestDb(object): "" ), 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' - '-c \\\dv': ( + '-c \\\\dv': ( 0, "", "Did not find any relations." ), } @@ -84,7 +84,7 @@ def test_negative(self): def test_psql_cmd(self): db = self.get_db() - res = db.psql_cmd('\\\dt') + res = db.psql_cmd('\\\\dt') assert 'List of relations' in res - res = db.psql_cmd('\\\dv') + res = db.psql_cmd('\\\\dv') assert res == 'Did not find any relations.' From 66d87eb664b76a30589bf5b0a916ac7690157bf9 Mon Sep 17 00:00:00 2001 From: leistnerova Date: Mon, 11 Feb 2019 09:23:57 +0100 Subject: [PATCH 50/55] Added negative test for psql_cmd --- tests/test_db.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_db.py b/tests/test_db.py index 8174731..970cf9a 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -49,6 +49,10 @@ class TestDb(object): '-c \\\\dv': ( 0, "", "Did not find any relations." ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c \\\\gg': ( + 1, "", "invalid command \\gg" + ), } files = {} @@ -88,3 +92,9 @@ def test_psql_cmd(self): assert 'List of relations' in res res = db.psql_cmd('\\\\dv') assert res == 'Did not find any relations.' + + def test_negative_cmd(self): + db = self.get_db() + with pytest.raises(Exception) as ex_info: + db.psql_cmd('\\\\gg') + assert 'invalid command' in str(ex_info.value) From db429e743271cfc06713f66ec018d006a9df95de Mon Sep 17 00:00:00 2001 From: roniezr <31003322+roniezr@users.noreply.github.com> Date: Wed, 13 Mar 2019 12:52:56 +0200 Subject: [PATCH 51/55] Add new function add_ip() (#114) * Add new function add_ip() * Add tests for the new added functions add_ip() * Fix fail tests * Fix add_ip() tests output Please enter the commit message for your changes. Lines starting * Fix Meni's CR comment --- rrmngmnt/network.py | 21 +++++++++++++++++++-- tests/test_network.py | 14 ++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 9eb93f7..f82c622 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -630,7 +630,7 @@ def if_up(self, nic): Returns: bool: True if setting nic up succeeded, false otherwise """ - cmd = "ip link set up %s" % nic + cmd = "ip link set {nic} up".format(nic=nic) rc, _, _ = self.host.run_command(shlex.split(cmd)) return not bool(rc) @@ -646,7 +646,7 @@ def if_down(self, nic, tcp_timeout=20, io_timeout=20): Returns: bool: True if setting nic down succeeded, false otherwise """ - cmd = "ip link set down %s" % nic + cmd = "ip link set {nic} down".format(nic=nic) rc, _, _ = self.host.run_command( command=shlex.split(cmd), tcp_timeout=tcp_timeout, @@ -654,6 +654,23 @@ def if_down(self, nic, tcp_timeout=20, io_timeout=20): ) return not bool(rc) + def add_ip(self, nic, ip, mask): + """ + Add IP address to interface + + Args: + nic (str): Interface name + ip (str): IP address to add + mask (str): IP netmask + + Returns: + bool: True if add IP was success, False otherwise + """ + cmd = "ip address add {ip}/{mask} dev {nic}".format( + ip=ip, mask=mask, nic=nic + ) + return not bool(self.host.run_command(command=shlex.split(cmd))[0]) + def is_connective(self, ping_timeout=20.0): """ Check if host network is connective via ping command diff --git a/tests/test_network.py b/tests/test_network.py index d2c2ba2..a184b58 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -183,12 +183,14 @@ class TestNetwork(object): "Permanent address: 44:1e:a1:73:3c:98", '' ), - 'ip link set up interface': True, - 'ip link set down interface': True, + 'ip link set interface up': True, + 'ip link set interface down': True, "ethtool -i eth0": (0, "driver: e1000", ""), "cat /sys/class/net/eth0/speed": (0, "1000", ""), "cat /sys/class/net/eth0/operstate": (0, "up", ""), "ping 1.2.3.4 -c 5 -s 10 -M do": (0, "something", ""), + 'ip address add 1.2.3.4/24 dev eth0': (0, "", ""), + 'ip address add 1.2.3.4/255.255.255.0 dev eth0': (0, "", ""), } files = { } @@ -269,6 +271,14 @@ def test_get_interface_status(self): def test_send_icmp(self): assert get_host().network.send_icmp('1.2.3.4', size="10") + def test_add_ip_with_bitmask(self): + assert get_host().network.add_ip(nic="eth0", ip="1.2.3.4", mask="24") + + def test_add_ip_with_subnet_mask(self): + assert get_host().network.add_ip( + nic="eth0", ip="1.2.3.4", mask="255.255.255.0" + ) + class TestHostNameCtl(object): From 4b4d9500a157889751fc10712a9cc9df9d84d975 Mon Sep 17 00:00:00 2001 From: Lukas Bednar Date: Wed, 30 Oct 2019 13:33:54 +0100 Subject: [PATCH 52/55] tests: pin to attrs-19.1.0 to enable tests execution (#116) https://github.com/AndreLouisCaron/pytest-docker/issues/39 https://github.com/python-attrs/attrs/pull/504 https://github.com/python-attrs/attrs/issues/307 Signed-off-by: Lukas Bednar --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index b87e66c..9252814 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,3 +2,4 @@ pytest>=2.8.7 pytest-cov pytest-docker docker-compose +attrs==19.1.0 From 910dc861cf689bd1cc9bf71c796eb2d357dddbac Mon Sep 17 00:00:00 2001 From: Jan Zmeskal Date: Thu, 21 Nov 2019 11:26:35 +0100 Subject: [PATCH 53/55] Add PlaybookRunner service (#117) * Add PlaybookRunner service * Add common.CommandReader * Add unit tests for common.CommandReader * Add unit tests for PlaybookRunner service * Docstrings for PlaybookRunner and its tests * Add ssh_common_args param to PlaybookRunner.run --- rrmngmnt/common.py | 56 +++++++ rrmngmnt/host.py | 5 + rrmngmnt/playbook_runner.py | 192 +++++++++++++++++++++++ rrmngmnt/resource.py | 8 +- rrmngmnt/ssh.py | 4 +- tests/common.py | 2 +- tests/test_common.py | 78 +++++++++- tests/test_playbook_runner.py | 286 ++++++++++++++++++++++++++++++++++ 8 files changed, 626 insertions(+), 5 deletions(-) create mode 100644 rrmngmnt/playbook_runner.py create mode 100644 tests/test_playbook_runner.py diff --git a/rrmngmnt/common.py b/rrmngmnt/common.py index 923e84b..ac90f9c 100644 --- a/rrmngmnt/common.py +++ b/rrmngmnt/common.py @@ -37,3 +37,59 @@ def normalize_string(data): if isinstance(data, six.text_type): data = data.encode('utf-8', errors='replace') return data + + +class CommandReader(object): + """ + This class is for gradual reading of commands output lines as they come in. + Each instance of CommandReader is tied to one command and one executor. + The executor calls the command only once the method read_lines is called. + After the execution of command finishes, CommandReader object may be + queried for return code, stdout and stderr of the command. + + Example usage: + my_host = Host("1.2.3.4") + my_host.users.append(RootUser("1234")) + my_executor = my_host.executor() + cr = CommandReader(my_executor, ['ansible-playbook', 'long_task.yml'] + for line in cr.read_lines(): + print(line) + """ + + def __init__(self, executor, cmd, cmd_input=None): + """ + Args: + executor (rrmngmnt.Executor): instance of rrmngmnt.Executor class + or one of its subclasses that executes provided command + cmd (list): Command to be executed + cmd_input(str): Input for the command + """ + self.executor = executor + self.cmd = cmd + self.cmd_input = cmd_input + self.rc = None + self.out = '' + self.err = '' + + def read_lines(self): + """ + Generator that yields lines of command output as they come to + underlying file handler. + + Yields: + str: Line of command's output stripped of newline character + """ + with self.executor.session() as ss: + command = ss.command(self.cmd) + with command.execute() as (in_, out, err): + if self.cmd_input: + in_.write(self.cmd_input) + in_.close() + while True: + line = out.readline() + self.out += line + if not line: + break + yield line.strip('\n') + self.rc = command.rc + self.err = err.read() diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 917c4e9..4bfe065 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -19,6 +19,7 @@ from rrmngmnt.network import Network from rrmngmnt.operatingsystem import OperatingSystem from rrmngmnt.package_manager import PackageManagerProxy +from rrmngmnt.playbook_runner import PlaybookRunner from rrmngmnt.resource import Resource from rrmngmnt.service import Systemd, SysVinit, InitCtl from rrmngmnt.storage import NFSService, LVMService @@ -460,6 +461,10 @@ def lvm(self): def fs(self): return FileSystem(self) + @property + def playbook(self): + return PlaybookRunner(self) + @property def ssh_public_key(self): return self.get_ssh_public_key() diff --git a/rrmngmnt/playbook_runner.py b/rrmngmnt/playbook_runner.py new file mode 100644 index 0000000..cd3a19c --- /dev/null +++ b/rrmngmnt/playbook_runner.py @@ -0,0 +1,192 @@ +import contextlib +import json +import os.path +import uuid + +from rrmngmnt.common import CommandReader +from rrmngmnt.resource import Resource +from rrmngmnt.service import Service + + +class PlaybookRunner(Service): + """ + Class for working with and especially executing Ansible playbooks on hosts. + On your Host instance, it might be accessed (similar to other services) by + playbook property. + + Example: + rc,out, err = my_host.playbook.run('long_task.yml') + + In such case, the default logger of this class (called PlaybookRunner) will + be used to log playbook's output. It will propagate events to handlers of + ancestor loggers. However, PlaybookRunner might also be directly + instantiated with instance of logging.Logger passed to logger parameter. + + Example: + my_runner = PlaybookRunner(my_host, logging.getLogger('playbook')) + rc, out, err = my_runner.run('long_task.yml') + + In that case, custom provided logger will be used instead. In both cases, + each log record will be prefixed with UUID generated specifically for that + one playbook execution. + """ + class LoggerAdapter(Resource.LoggerAdapter): + + def process(self, msg, kwargs): + return "[%s] %s" % (self.extra['self'].short_run_uuid, msg), kwargs + + tmp_dir = "/tmp" + binary = "ansible-playbook" + extra_vars_file = "extra_vars.json" + default_inventory_name = "inventory" + default_inventory_content = "localhost ansible_connection=local" + ssh_common_args_param = "--ssh-common-args" + check_mode_param = "--check" + + def __init__(self, host, logger=None): + """ + Args: + host (rrmngmnt.Host): Underlying host for this service + logger (logging.Logger): Alternate logger for Ansible output + """ + super(PlaybookRunner, self).__init__(host) + if logger: + self.set_logger(logger) + self.run_uuid = uuid.uuid4() + self.short_run_uuid = str(self.run_uuid).split('-')[0] + self.tmp_exec_dir = None + self.cmd = [self.binary] + self.rc = None + self.out = None + self.err = None + + @contextlib.contextmanager + def _exec_dir(self): + """ + Context manager that makes sure that for each execution of playbook, + temporary directory (whose name is the same as run's UUID) is created + on the host and removed afterwards. + """ + exec_dir_path = os.path.join(self.tmp_dir, self.short_run_uuid) + self.host.fs.rmdir(exec_dir_path) + self.host.fs.mkdir(exec_dir_path) + self.tmp_exec_dir = exec_dir_path + try: + yield + finally: + self.tmp_exec_dir = None + self.host.fs.rmdir(exec_dir_path) + + def _upload_file(self, file_): + file_path_on_host = os.path.join( + self.tmp_exec_dir, os.path.basename(file_) + ) + self.host.fs.put(path_src=file_, path_dst=file_path_on_host) + return file_path_on_host + + def _dump_vars_to_json_file(self, vars_): + file_path_on_host = os.path.join( + self.tmp_exec_dir, self.extra_vars_file + ) + self.host.fs.create_file( + content=json.dumps(vars_), path=file_path_on_host + ) + return file_path_on_host + + def _generate_default_inventory(self): + file_path_on_host = os.path.join( + self.tmp_exec_dir, self.default_inventory_name + ) + self.host.fs.create_file( + content=self.default_inventory_content, + path=file_path_on_host + ) + return file_path_on_host + + def run( + self, playbook, extra_vars=None, vars_files=None, inventory=None, + verbose_level=1, run_in_check_mode=False, ssh_common_args=None, + ): + """ + Run Ansible playbook on host + + Args: + playbook (str): Path to playbook you want to execute (on your + machine) + extra_vars (dict): Dictionary of extra variables that are to be + passed to playbook execution. They will be dumped to JSON file + and included using -e@ parameter + vars_files (list): List of additional variable files to be included + using -e@ parameter. If one variable is specified both in + extra_vars and in one of the vars_files, the one in vars_files + takes precedence. + inventory (str): Path to an inventory file (on your machine) to be + used for playbook execution. If none is provided, default + inventory including only localhost will be generated and used + verbose_level (int): How much should playbook be verbose. Possible + values are 1 through 5 with 1 being the most quiet and 5 being + the most verbose + run_in_check_mode (bool): If True, playbook will not actually be + executed, but instead run with --check parameter + ssh_common_args (list): List of options that will extend (not + replace) the list of default options that Ansible uses when + calling ssh/sftp/scp. Example: ["-o StrictHostKeyChecking=no", + "-o UserKnownHostsFile=/dev/null"] + + Returns: + tuple: tuple of (rc, out, err) + """ + self.logger.info( + "Running playbook {} on {}".format( + os.path.basename(playbook), + self.host.fqdn + ) + ) + + with self._exec_dir(): + + if extra_vars: + self.cmd.append( + "-e@{}".format(self._dump_vars_to_json_file(extra_vars)) + ) + + if vars_files: + for f in vars_files: + self.cmd.append("-e@{}".format(self._upload_file(f))) + + self.cmd.append("-i") + if inventory: + self.cmd.append(self._upload_file(inventory)) + else: + self.cmd.append(self._generate_default_inventory()) + + self.cmd.append("-{}".format("v" * verbose_level)) + + if run_in_check_mode: + self.cmd.append(self.check_mode_param) + + if ssh_common_args: + self.cmd.append( + "{}={}".format( + self.ssh_common_args_param, " ".join(ssh_common_args) + ) + ) + + self.cmd.append(self._upload_file(playbook)) + + self.logger.debug("Executing: {}".format(" ".join(self.cmd))) + + playbook_reader = CommandReader(self.host.executor(), self.cmd) + for line in playbook_reader.read_lines(): + self.logger.debug(line) + self.rc, self.out, self.err = ( + playbook_reader.rc, + playbook_reader.out, + playbook_reader.err + ) + + self.logger.debug( + "Ansible playbook finished with RC: {}".format(self.rc) + ) + + return self.rc, self.out, self.err diff --git a/rrmngmnt/resource.py b/rrmngmnt/resource.py index add7855..9764a72 100644 --- a/rrmngmnt/resource.py +++ b/rrmngmnt/resource.py @@ -16,8 +16,14 @@ def warn(self, *args, **kwargs): def __init__(self): super(Resource, self).__init__() logger = logging.getLogger(self.__class__.__name__) - self._logger_adapter = self.LoggerAdapter(logger, {'self': self}) + self.set_logger(logger) @property def logger(self): return self._logger_adapter + + def set_logger(self, logger): + if isinstance(logger, logging.Logger): + self._logger_adapter = self.LoggerAdapter(logger, {'self': self}) + elif isinstance(logger, logging.LoggerAdapter): + self._logger_adapter = logger diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 4ce2a23..c15c308 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -21,12 +21,12 @@ class RemoteExecutor(Executor): Any resource which provides SSH service. This class is meant to replace our current utilities.machine.LinuxMachine - classs. This allows you to lower access to communicate with ssh. + class. This allows you to lower access to communicate with ssh. Like a live interaction, getting rid of True/False results, and mixing stdout with stderr. You can still use use 'run_cmd' method if you don't care. - But I would recommed you to work like this: + But I would recommend you to work like this: """ TCP_TIMEOUT = 10.0 diff --git a/tests/common.py b/tests/common.py index 11be733..77f72e6 100644 --- a/tests/common.py +++ b/tests/common.py @@ -106,8 +106,8 @@ def run(self, input_, timeout=None): @contextlib.contextmanager def execute(self, bufsize=-1, timeout=None): rc, out, err = self._ss.get_data(self.cmd) - yield six.StringIO(), six.StringIO(out), six.StringIO(err) self._rc = rc + yield six.StringIO(), six.StringIO(out), six.StringIO(err) def __init__(self, user, address): super(FakeExecutor, self).__init__(user) diff --git a/tests/test_common.py b/tests/test_common.py index 00b6bfa..b03aeb8 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- +import types + import pytest import netaddr -from rrmngmnt import common +from rrmngmnt import common, Host, User +from .common import FakeExecutorFactory def test_fqdn2ip_positive(): @@ -13,3 +16,76 @@ def test_fqdn2ip_negative(): with pytest.raises(Exception) as ex_info: common.fqdn2ip('github.or') assert 'github.or' in str(ex_info.value) + + +class TestCommandReader(object): + + data = { + 'cat shopping_list.txt': (0, 'bananas\nmilk\nhuge blender', ''), + 'cat milk_shake_recipe.txt': ( + 1, '', 'cat: milk_shake_recipe.txt: No such file or directory' + ), + } + files = {} + + @classmethod + @pytest.fixture(scope='class') + def fake_host(cls): + fh = Host('1.1.1.1') + fh.add_user(User('root', '11111')) + fh.executor_factory = FakeExecutorFactory(cls.data, cls.files) + return fh + + def test_return_type(self, fake_host): + """ Test that CommandReader returns generator type """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + ret = cmd_reader.read_lines() + + assert isinstance(ret, types.GeneratorType) + + expected_output = self.data[cmd][1].split('\n') + for i in range(len(expected_output)): + assert next(ret) == expected_output[i] + + with pytest.raises(StopIteration): + next(ret) + + def test_iterate_over_output(self, fake_host): + """ + Test that we can iterate over CommandReader's output using for loop + """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + expected_output = self.data[cmd][1].split('\n') + cmd_reader_output = [] + + for line in cmd_reader.read_lines(): + cmd_reader_output.append(line) + + assert cmd_reader_output == expected_output + + def test_return_code(self, fake_host): + """ Test that rc of command is captured by CommandReader """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + + assert cmd_reader.rc is None + + for line in cmd_reader.read_lines(): + pass + + assert not cmd_reader.rc + + def test_stderr(self, fake_host): + """ Test that error output is captured by CommandReader """ + cmd = 'cat milk_shake_recipe.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + + assert not cmd_reader.err + + for line in cmd_reader.read_lines(): + pass + + assert cmd_reader.rc + assert cmd_reader.err diff --git a/tests/test_playbook_runner.py b/tests/test_playbook_runner.py new file mode 100644 index 0000000..66d7f90 --- /dev/null +++ b/tests/test_playbook_runner.py @@ -0,0 +1,286 @@ +import os.path + +import pytest + +from rrmngmnt import Host, User +from rrmngmnt.playbook_runner import PlaybookRunner +from .common import FakeExecutorFactory + + +class PlaybookRunnerBase(object): + + # The fake run UUID will be used instead of unique ID that's auto-generated + # for each playbook execution + fake_run_uuid = '123' + + playbook_name = 'test.yml' + playbook_content = '' + vars_file_name = 'my_vars.yml' + vars_file_content = '' + inventory_name = 'my_inventory' + inventory_content = '' + ssh_no_strict_host_key_checking = "-o StrictHostKeyChecking=no" + + tmp_dir = os.path.join(PlaybookRunner.tmp_dir, fake_run_uuid) + + success = (0, '', '') + failure = (1, '', '') + + data = { + # Filesystem-related operations + 'rm -rf {}'.format(tmp_dir): success, + 'mkdir {}'.format(tmp_dir): success, + '[ -d {tmp_dir}/{playbook} ]'.format( + tmp_dir=tmp_dir, playbook=playbook_name + ): failure, + '[ -d {tmp_dir}/{vars_file} ]'.format( + tmp_dir=tmp_dir, vars_file=vars_file_name + ): failure, + '[ -d {tmp_dir}/{inventory} ]'.format( + tmp_dir=tmp_dir, inventory=inventory_name + ): failure, + # Actual execution of ansible-playbook + # Basic scenario + '{bin} -i {tmp_dir}/{inventory} -v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Extra vars have been provided + '{bin} -e@{tmp_dir}/{extra_vars} -i {tmp_dir}/{inventory} ' + '-v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + extra_vars=PlaybookRunner.extra_vars_file, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # File with additional variables has been provided + '{bin} -e@{tmp_dir}/{vars_file} -i {tmp_dir}/{inventory} ' + '-v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + vars_file=vars_file_name, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Custom inventory has been provided + '{bin} -i {tmp_dir}/{inventory} -v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=inventory_name, + playbook=playbook_name + ): success, + # Verbosity has been increased to max + '{bin} -i {tmp_dir}/{inventory} -vvvvv {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Running in check mode + '{bin} -i {tmp_dir}/{inventory} -v {check_mode_param} ' + '{tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + check_mode_param=PlaybookRunner.check_mode_param, + playbook=playbook_name + ): success, + # Running with extended SSH common args + '{bin} -i {tmp_dir}/{inventory} ' + '-v "{ssh_common_args_param}={ssh_common_args}" ' + '{tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + ssh_common_args_param=PlaybookRunner.ssh_common_args_param, + ssh_common_args=ssh_no_strict_host_key_checking, + playbook=playbook_name + ): success, + } + + @classmethod + @pytest.fixture(scope='class') + def fake_host(cls): + fh = Host('1.1.1.1') + fh.add_user(User('root', '11111')) + fh.executor_factory = FakeExecutorFactory(cls.data, cls.files) + return fh + + @pytest.fixture() + def fake_playbook(self, tmpdir): + fp = tmpdir.join(self.playbook_name) + fp.write(self.playbook_content) + return str(fp) + + @pytest.fixture() + def playbook_runner(self, fake_host): + playbook_runner = PlaybookRunner(fake_host) + playbook_runner.short_run_uuid = self.fake_run_uuid + return playbook_runner + + def check_files_on_host(self, files=None): + """ + Check that all files provided to files parameter (and only those) have + been "copied" to our imaginary host. In reality, they should not be + present on host once the playbook's execution is done. However here + we'll use the fact that our fake host does not really implements file + removal. Because of this, in the end of the test case, we can check + that files that should have been copied to the host (by using + FileSystem service) have actually been sent there. + + Args: + files (list): List of files that should have been copied to the + host. Don't include test playbook into this list since its + presence is implicitly expected. You can also provide only one + file as a string. + + Returns: + bool: True if files expected on host and those present match, + False otherwise + """ + if files is None: + files = [] + if isinstance(files, str): + files = [files] + expected_files = [os.path.join(self.tmp_dir, self.playbook_name)] + expected_files.extend(files) + return sorted(expected_files) == sorted(list(self.files.keys())) + + +class TestBasic(PlaybookRunnerBase): + + files = {} + + def test_basic_scenario(self, playbook_runner, fake_playbook): + """ User has provided only playbook """ + rc, _, _ = playbook_runner.run(playbook=fake_playbook) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestExtraVars(PlaybookRunnerBase): + + files = {} + + def test_extra_vars(self, playbook_runner, fake_playbook): + """ User has provided extra vars as a dictionary """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + extra_vars={ + "greetings": "hello", + } + ) + assert not rc + assert self.check_files_on_host( + [ + os.path.join( + self.tmp_dir, PlaybookRunner.default_inventory_name + ), + os.path.join(self.tmp_dir, PlaybookRunner.extra_vars_file) + ] + ) + + +class TestVarsFile(PlaybookRunnerBase): + + files = {} + + @pytest.fixture() + def fake_vars_file(self, tmpdir): + fvf = tmpdir.join(self.vars_file_name) + fvf.write(self.vars_file_content) + return str(fvf) + + def test_vars_file(self, playbook_runner, fake_playbook, fake_vars_file): + """ User has provided YAML file with custom variables """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + vars_files=[fake_vars_file] + ) + assert not rc + assert self.check_files_on_host( + [ + os.path.join( + self.tmp_dir, PlaybookRunner.default_inventory_name + ), + os.path.join(self.tmp_dir, self.vars_file_name) + ] + ) + + +class TestInventory(PlaybookRunnerBase): + + files = {} + + @pytest.fixture() + def fake_inventory(self, tmpdir): + fi = tmpdir.join(self.inventory_name) + fi.write(self.inventory_content) + return str(fi) + + def test_inventory(self, playbook_runner, fake_playbook, fake_inventory): + """ User has provided custom inventory instead of the default one """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + inventory=fake_inventory + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, self.inventory_name) + ) + + +class TestVerbosity(PlaybookRunnerBase): + + files = {} + + def test_max_verbosity(self, playbook_runner, fake_playbook): + """ User has increased verbosity to maximum level """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + verbose_level=5 + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestCheckMode(PlaybookRunnerBase): + + files = {} + + def test_check_mode(self, playbook_runner, fake_playbook): + """ User is running the playbook with --check param """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + run_in_check_mode=True + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestSSHCommonArgs(PlaybookRunnerBase): + + files = {} + + def test_no_strict_host_key_checking(self, playbook_runner, fake_playbook): + """ + User has provided custom SSH argument that extend default Ansible SSH + arguments + """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + ssh_common_args=[self.ssh_no_strict_host_key_checking] + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) From e12c8b01b2e63126885a30f6223261f798c20179 Mon Sep 17 00:00:00 2001 From: roniezr <31003322+roniezr@users.noreply.github.com> Date: Fri, 10 Jan 2020 10:28:48 +0200 Subject: [PATCH 54/55] Support Python3 (#118) * Python3 support Change Output & Error of a running command to always return 'str' instead of 'bytes', to be compatible with python2 * Add tests to 'normalize_string' function --- rrmngmnt/common.py | 2 -- rrmngmnt/host.py | 2 +- tests/test_common.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/rrmngmnt/common.py b/rrmngmnt/common.py index ac90f9c..884506b 100644 --- a/rrmngmnt/common.py +++ b/rrmngmnt/common.py @@ -34,8 +34,6 @@ def normalize_string(data): """ if isinstance(data, six.binary_type): data = data.decode('utf-8', errors='replace') - if isinstance(data, six.text_type): - data = data.encode('utf-8', errors='replace') return data diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 4bfe065..e0ec769 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -147,7 +147,7 @@ def get_power_manager(self, pm_type=None): (pm_type, self) ) else: - return self._power_managers.values()[0] + return list(self._power_managers.values())[0] raise Exception("No PM is associated with the host %s" % self) def get_user(self, name): diff --git a/tests/test_common.py b/tests/test_common.py index b03aeb8..63f09c4 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -5,6 +5,7 @@ import netaddr from rrmngmnt import common, Host, User from .common import FakeExecutorFactory +import six def test_fqdn2ip_positive(): @@ -89,3 +90,45 @@ def test_stderr(self, fake_host): assert cmd_reader.rc assert cmd_reader.err + + +def test_normalize_string_bytes_input(): + """ + Test 'normalize_string' function with 'bytes' input + + In python3 we want to convert bytes() to str(), + to eliminate TypeError exception when mixing between bytes & str + at the calling function + In python2 there is no meaning if the output will by bytes() or str() + python will not raise a TypeError exception when mixing between them + """ + if six.PY3: + assert type(common.normalize_string(data=bytes())) == str + + +def test_normalize_string_str_input(): + """ + Test 'normalize_string' function with 'str' input + + Keep the str type in python3 + Convert the str type to unicode in python2 + """ + try: + expected_type = unicode # Python2 + except NameError: + expected_type = str # Python3 + + assert type(common.normalize_string(data=str())) == expected_type + + +def test_normalize_string_unicode_input(): + """ + Test 'normalize_string' function with 'unicode' input (PY2 only) + + In python2 unicode input should not be converted + 'unicode' is not supported at python3 + """ + if six.PY2: + assert type( + common.normalize_string(data=unicode()) # noqa: F821 + ) == unicode # noqa: F821 From d9114fccd9d611864b13f51b06ef596e2faaa077 Mon Sep 17 00:00:00 2001 From: ipinto Date: Wed, 15 Jan 2020 22:17:37 +0200 Subject: [PATCH 55/55] Set TCP connection timeout --- rrmngmnt/ssh.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index c15c308..32811dd 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -14,6 +14,7 @@ ID_RSA_PRV = os.path.join("%s", ".ssh/id_rsa") CONNECTIVITY_TIMEOUT = 600 CONNECTIVITY_SAMPLE_TIME = 20 +TCP_CONNECTION_TIMEOUT = 20 class RemoteExecutor(Executor): @@ -266,7 +267,8 @@ def is_connective(self, tcp_timeout=20.0): def wait_for_connectivity_state( self, positive, timeout=CONNECTIVITY_TIMEOUT, - sample_time=CONNECTIVITY_SAMPLE_TIME + sample_time=CONNECTIVITY_SAMPLE_TIME, + tcp_connection_timeout=TCP_CONNECTION_TIMEOUT ): """ Wait until address will be connective or not via ssh @@ -275,6 +277,7 @@ def wait_for_connectivity_state( positive (bool): Wait for the positive or negative connective state timeout (int): Wait timeout sample_time (int): Sample the ssh each sample_time seconds + tcp_connection_timeout (int): TCP connection timeout Returns: bool: True, if positive and ssh is connective or negative and ssh @@ -282,7 +285,7 @@ def wait_for_connectivity_state( """ reachable = "unreachable" if positive else "reachable" timeout_counter = 0 - while self.is_connective() != positive: + while self.is_connective(tcp_timeout=tcp_connection_timeout) != positive: if timeout_counter > timeout: self.logger.error( "Address %s is still %s via ssh, after %s seconds",