From 0de475c0d3621f71499116b0da13f734e89ddc21 Mon Sep 17 00:00:00 2001 From: ArbelNathan Date: Mon, 15 May 2023 17:09:12 +0300 Subject: [PATCH 1/7] add fence sidecar to controller Signed-off-by: ArbelNathan --- controllers/scripts/csi_general/csi_pb2.sh | 2 ++ .../servers/csi/controller_server_manager.py | 5 ++++- controllers/servers/csi/fence.py | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 controllers/servers/csi/fence.py diff --git a/controllers/scripts/csi_general/csi_pb2.sh b/controllers/scripts/csi_general/csi_pb2.sh index d485fabc4..9724f0001 100755 --- a/controllers/scripts/csi_general/csi_pb2.sh +++ b/controllers/scripts/csi_general/csi_pb2.sh @@ -12,7 +12,9 @@ cd ./proto/${PB2_DIR} curl -O https://raw.githubusercontent.com/container-storage-interface/spec/${CSI_VERSION}/csi.proto curl -O https://raw.githubusercontent.com/IBM/csi-volume-group/${VG_VERSION}/volumegroup/volumegroup.proto curl -O https://raw.githubusercontent.com/csi-addons/spec/v0.2.0/replication/replication.proto +curl -O https://raw.githubusercontent.com/csi-addons/spec/v0.2.0/fence/fence.proto sed -i 's|github.com/container-storage-interface/spec/lib/go/csi/csi.proto|csi_general/csi.proto|g' replication.proto +sed -i 's|github.com/container-storage-interface/spec/lib/go/csi/csi.proto|csi_general/csi.proto|g' fence.proto cd - python -m grpc_tools.protoc --proto_path=proto \ diff --git a/controllers/servers/csi/controller_server_manager.py b/controllers/servers/csi/controller_server_manager.py index e5458403a..85eff8cc3 100644 --- a/controllers/servers/csi/controller_server_manager.py +++ b/controllers/servers/csi/controller_server_manager.py @@ -3,7 +3,7 @@ from concurrent import futures import grpc -from csi_general import csi_pb2_grpc, replication_pb2_grpc, volumegroup_pb2_grpc +from csi_general import csi_pb2_grpc, replication_pb2_grpc, volumegroup_pb2_grpc, fence_pb2_grpc from controllers.common.config import config from controllers.common.csi_logger import get_stdout_logger @@ -11,6 +11,7 @@ from controllers.servers.csi.addons_server import ReplicationControllerServicer from controllers.servers.csi.csi_controller_server import CSIControllerServicer from controllers.servers.csi.volume_group_server import VolumeGroupControllerServicer +from controllers.servers.csi.fence import FenceControllerServicer logger = get_stdout_logger() @@ -26,6 +27,7 @@ def __init__(self, array_endpoint): self.csi_servicer = CSIControllerServicer() self.replication_servicer = ReplicationControllerServicer() self.volume_group_servicer = VolumeGroupControllerServicer() + self.fence_servicer = FenceControllerServicer() def start_server(self): max_workers = get_max_workers_count() @@ -35,6 +37,7 @@ def start_server(self): csi_pb2_grpc.add_IdentityServicer_to_server(self.csi_servicer, controller_server) replication_pb2_grpc.add_ControllerServicer_to_server(self.replication_servicer, controller_server) volumegroup_pb2_grpc.add_ControllerServicer_to_server(self.volume_group_servicer, controller_server) + fence_pb2_grpc.add_FenceControllerServicer_to_server(self.fence_servicer, controller_server) # bind the server to the port defined above # controller_server.add_insecure_port('[::]:{}'.format(self.server_port)) diff --git a/controllers/servers/csi/fence.py b/controllers/servers/csi/fence.py new file mode 100644 index 000000000..184df5e41 --- /dev/null +++ b/controllers/servers/csi/fence.py @@ -0,0 +1,17 @@ +from csi_general import fence_pb2_grpc, fence_pb2 + +from controllers.common.csi_logger import get_stdout_logger +from controllers.servers.csi.decorators import csi_method + +logger = get_stdout_logger() + +class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): + @csi_method(error_response_type=fence_pb2.FenceClusterNetworkResponse, lock_request_attribute="cidrs") + def FenceClusterNetwork(self, request, context): + logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) + logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) + + @csi_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse, lock_request_attribute="cidrs") + def UnfenceClusterNetwork(self, request, context): + logger.debug("UnfenceClusterNetwork parameters : {}".format(request.parameters)) + logger.debug("UnfenceClusterNetwork cidrs : {}".format(request.cidrs)) \ No newline at end of file From 24bfa53ac7f64a516bccee2d37e4b1b6b667e5d7 Mon Sep 17 00:00:00 2001 From: ArbelNathan Date: Tue, 23 May 2023 16:38:33 +0300 Subject: [PATCH 2/7] merge with csi_addon brunch Signed-off-by: ArbelNathan --- controllers/servers/__init__.py | 0 .../servers/csi/{ => csi_addons_server}/fence.py | 5 +++++ controllers/servers/csi/main.py | 16 +++++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 controllers/servers/__init__.py rename controllers/servers/csi/{ => csi_addons_server}/fence.py (88%) diff --git a/controllers/servers/__init__.py b/controllers/servers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/controllers/servers/csi/fence.py b/controllers/servers/csi/csi_addons_server/fence.py similarity index 88% rename from controllers/servers/csi/fence.py rename to controllers/servers/csi/csi_addons_server/fence.py index 184df5e41..6d4626eec 100644 --- a/controllers/servers/csi/fence.py +++ b/controllers/servers/csi/csi_addons_server/fence.py @@ -10,6 +10,11 @@ class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): def FenceClusterNetwork(self, request, context): logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) + """ + 1. get ogA's pools + 2. move pools to ogB + 3. remove all mapping from ogA + """ @csi_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse, lock_request_attribute="cidrs") def UnfenceClusterNetwork(self, request, context): diff --git a/controllers/servers/csi/main.py b/controllers/servers/csi/main.py index 60c1a5682..b90705630 100644 --- a/controllers/servers/csi/main.py +++ b/controllers/servers/csi/main.py @@ -1,19 +1,19 @@ import os from argparse import ArgumentParser -from threading import Thread -from concurrent import futures -import grpc from concurrent import futures +from threading import Thread -from csi_general import csi_pb2_grpc, volumegroup_pb2_grpc, identity_pb2_grpc, replication_pb2_grpc +import grpc +from csi_general import csi_pb2_grpc, volumegroup_pb2_grpc, identity_pb2_grpc, replication_pb2_grpc, fence_pb2_grpc from controllers.common.csi_logger import set_log_level from controllers.common.settings import CSI_CONTROLLER_SERVER_WORKERS -from controllers.servers.csi.server_manager import ServerManager from controllers.servers.csi.controller_server.csi_controller_server import CSIControllerServicer from controllers.servers.csi.controller_server.volume_group_server import VolumeGroupControllerServicer -from controllers.servers.csi.csi_addons_server.replication_controller_servicer import ReplicationControllerServicer +from controllers.servers.csi.csi_addons_server.fence import FenceControllerServicer from controllers.servers.csi.csi_addons_server.identity_controller_servicer import IdentityControllerServicer +from controllers.servers.csi.csi_addons_server.replication_controller_servicer import ReplicationControllerServicer +from controllers.servers.csi.server_manager import ServerManager def main(): @@ -56,8 +56,10 @@ def _add_csi_controller_servicers(controller_server): def _add_csi_addons_servicers(csi_addons_server): replication_servicer = ReplicationControllerServicer() identity_servicer = IdentityControllerServicer() + fence_servicer = FenceControllerServicer() replication_pb2_grpc.add_ControllerServicer_to_server(replication_servicer, csi_addons_server) identity_pb2_grpc.add_IdentityServicer_to_server(identity_servicer, csi_addons_server) + fence_pb2_grpc.add_FenceControllerServicer_to_server(fence_servicer, csi_addons_server) return csi_addons_server @@ -66,7 +68,7 @@ def _start_servers(csi_controller_server_manager, csi_addons_server_manager): csi_controller_server_manager.start_server, csi_addons_server_manager.start_server) for server_function in servers: - thread = Thread(target=server_function,) + thread = Thread(target=server_function, ) thread.start() From 69d77405a9d8ad3bba4a5b78b8ed7dd3869edb75 Mon Sep 17 00:00:00 2001 From: ArbelNathan Date: Mon, 12 Jun 2023 18:14:17 +0300 Subject: [PATCH 3/7] add fence servicer Signed-off-by: ArbelNathan --- .../servers/csi/csi_addons_server/fence.py | 22 ------- .../fence_controller_servicer.py | 44 ++++++++++++++ controllers/servers/csi/decorators.py | 2 + controllers/servers/csi/main.py | 2 +- .../fence_controller_servicer_test.py | 58 +++++++++++++++++++ 5 files changed, 105 insertions(+), 23 deletions(-) delete mode 100644 controllers/servers/csi/csi_addons_server/fence.py create mode 100644 controllers/servers/csi/csi_addons_server/fence_controller_servicer.py create mode 100644 controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py diff --git a/controllers/servers/csi/csi_addons_server/fence.py b/controllers/servers/csi/csi_addons_server/fence.py deleted file mode 100644 index 6d4626eec..000000000 --- a/controllers/servers/csi/csi_addons_server/fence.py +++ /dev/null @@ -1,22 +0,0 @@ -from csi_general import fence_pb2_grpc, fence_pb2 - -from controllers.common.csi_logger import get_stdout_logger -from controllers.servers.csi.decorators import csi_method - -logger = get_stdout_logger() - -class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): - @csi_method(error_response_type=fence_pb2.FenceClusterNetworkResponse, lock_request_attribute="cidrs") - def FenceClusterNetwork(self, request, context): - logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) - logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) - """ - 1. get ogA's pools - 2. move pools to ogB - 3. remove all mapping from ogA - """ - - @csi_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse, lock_request_attribute="cidrs") - def UnfenceClusterNetwork(self, request, context): - logger.debug("UnfenceClusterNetwork parameters : {}".format(request.parameters)) - logger.debug("UnfenceClusterNetwork cidrs : {}".format(request.cidrs)) \ No newline at end of file diff --git a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py new file mode 100644 index 000000000..affc23823 --- /dev/null +++ b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py @@ -0,0 +1,44 @@ +from csi_general import fence_pb2_grpc, fence_pb2 + +from controllers.array_action.storage_agent import get_agent, detect_array_type +from controllers.common.csi_logger import get_stdout_logger +from controllers.servers import utils +from controllers.servers.csi.decorators import csi_method + +logger = get_stdout_logger() + + +def _is_already_handled(mediator, fence_ownership_group): + return mediator.is_fenced(fence_ownership_group) + + +def _fence_cluster_network(mediator, fence_ownership_group, unfence_ownership_group): + mediator.fence(fence_ownership_group, unfence_ownership_group) + + +def handle_fencing(request): + fence_ownership_group = request.parameters["fenceToken"] + unfence_ownership_group = request.parameters["unfenceToken"] + + connection_info = utils.get_array_connection_info_from_secrets(request.secrets) + array_type = detect_array_type(connection_info.array_addresses) + with get_agent(connection_info, array_type).get_mediator() as mediator: + # idempotence - check if the fence_ownership_group is already fenced (no pools in the og) + if _is_already_handled(mediator, fence_ownership_group): + return fence_pb2.FenceClusterNetworkResponse() + _fence_cluster_network(mediator, fence_ownership_group, unfence_ownership_group) + return fence_pb2.FenceClusterNetworkResponse() + + +class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): + @csi_method(error_response_type=fence_pb2.FenceClusterNetworkResponse, lock_request_attribute="parameters") + def FenceClusterNetwork(self, request, context): + logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) + logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) + return handle_fencing(request) + + @csi_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse, lock_request_attribute="parameters") + def UnfenceClusterNetwork(self, request, context): + logger.debug("UnfenceClusterNetwork parameters : {}".format(request.parameters)) + logger.debug("UnfenceClusterNetwork cidrs : {}".format(request.cidrs)) + return handle_fencing(request) diff --git a/controllers/servers/csi/decorators.py b/controllers/servers/csi/decorators.py index 819416dce..957724ebe 100644 --- a/controllers/servers/csi/decorators.py +++ b/controllers/servers/csi/decorators.py @@ -15,6 +15,8 @@ def csi_method(error_response_type, lock_request_attribute=''): @decorator def call_csi_method(controller_method, servicer, request, context): lock_id = getattr(request, lock_request_attribute, None) + if isinstance(lock_id, dict): + lock_id = lock_id.get('fenceToken', '') return _set_sync_lock(lock_id, lock_request_attribute, error_response_type, controller_method, servicer, request, context) diff --git a/controllers/servers/csi/main.py b/controllers/servers/csi/main.py index b90705630..8d0bced8d 100644 --- a/controllers/servers/csi/main.py +++ b/controllers/servers/csi/main.py @@ -10,7 +10,7 @@ from controllers.common.settings import CSI_CONTROLLER_SERVER_WORKERS from controllers.servers.csi.controller_server.csi_controller_server import CSIControllerServicer from controllers.servers.csi.controller_server.volume_group_server import VolumeGroupControllerServicer -from controllers.servers.csi.csi_addons_server.fence import FenceControllerServicer +from controllers.servers.csi.csi_addons_server.fence_controller_servicer import FenceControllerServicer from controllers.servers.csi.csi_addons_server.identity_controller_servicer import IdentityControllerServicer from controllers.servers.csi.csi_addons_server.replication_controller_servicer import ReplicationControllerServicer from controllers.servers.csi.server_manager import ServerManager diff --git a/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py b/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py new file mode 100644 index 000000000..1e869abc4 --- /dev/null +++ b/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py @@ -0,0 +1,58 @@ +import unittest + +import grpc +from mock import Mock, MagicMock + +from controllers.servers.csi.csi_addons_server.fence_controller_servicer import FenceControllerServicer +from controllers.tests import utils +from controllers.tests.common.test_settings import SECRET +from controllers.tests.controller_server.common import mock_array_type, mock_get_agent, mock_mediator + +FENCE_SERVER_PATH = "controllers.servers.csi.csi_addons_server.fence_controller_servicer" + + +class TestFenceControllerServicer(unittest.TestCase): + + def setUp(self): + self.servicer = FenceControllerServicer() + self.request = Mock() + self.context = utils.FakeContext() + mock_array_type(self, FENCE_SERVER_PATH) + + self.mediator = mock_mediator() + + self.storage_agent = MagicMock() + mock_get_agent(self, FENCE_SERVER_PATH) + + self.request.secrets = SECRET + self.request.parameters = {"fenceToken": "fenceToken", "unfenceToken": "unfenceToken"} + self.request.cidrs = ["0.0.0.0/32"] + self.mediator.is_fenced.return_value = False + + def test_fence_succeeds(self): + self.servicer.FenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.OK, self.context.code) + self.mediator.fence.assert_called_once_with("fenceToken", "unfenceToken") + + def test_fence_fails(self): + self.mediator.fence.side_effect = Exception("fence failed") + self.servicer.FenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.INTERNAL, self.context.code) + self.mediator.fence.assert_called_once_with("fenceToken", "unfenceToken") + + def test_fence_already_fenced(self): + self.mediator.is_fenced.return_value = True + self.servicer.FenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.OK, self.context.code) + self.mediator.fence.assert_not_called() + + def test_unfence_succeeds(self): + self.servicer.UnfenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.OK, self.context.code) + self.mediator.fence.assert_called_once_with("fenceToken", "unfenceToken") + + def test_unfence_already_fenced(self): + self.mediator.is_fenced.return_value = True + self.servicer.UnfenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.OK, self.context.code) + self.mediator.fence.assert_not_called() From fab8432304a44a369ad5910f2a6bff55f1cb3270 Mon Sep 17 00:00:00 2001 From: ArbelNathan Date: Wed, 14 Jun 2023 14:11:13 +0300 Subject: [PATCH 4/7] add fence capabilities and implementation in mediator Signed-off-by: ArbelNathan --- .../array_action/array_mediator_svc.py | 50 +++++++++++++++++-- controllers/array_action/fence_interface.py | 37 ++++++++++++++ .../fence_controller_servicer.py | 10 ++-- .../identity_controller_servicer.py | 9 +++- controllers/servers/csi/decorators.py | 10 +++- .../tests/array_action/svc/fence_svc_test.py | 45 +++++++++++++++++ controllers/tests/common/test_settings.py | 6 +++ 7 files changed, 158 insertions(+), 9 deletions(-) create mode 100644 controllers/array_action/fence_interface.py create mode 100644 controllers/tests/array_action/svc/fence_svc_test.py diff --git a/controllers/array_action/array_mediator_svc.py b/controllers/array_action/array_mediator_svc.py index ea9e216f6..482fdc59d 100644 --- a/controllers/array_action/array_mediator_svc.py +++ b/controllers/array_action/array_mediator_svc.py @@ -14,6 +14,7 @@ import controllers.servers.settings as controller_settings from controllers.array_action.array_action_types import Volume, Snapshot, Replication, Host, VolumeGroup, ThinVolume from controllers.array_action.array_mediator_abstract import ArrayMediatorAbstract +from controllers.array_action.fence_interface import FenceInterface from controllers.array_action.utils import ClassProperty, convert_scsi_id_to_nguid from controllers.array_action.volume_group_interface import VolumeGroupInterface from controllers.common import settings as common_settings @@ -99,7 +100,7 @@ def _get_space_efficiency_kwargs(space_efficiency): def _is_space_efficiency_matches_source(parameter_space_efficiency, array_space_efficiency): return (not parameter_space_efficiency and array_space_efficiency == common_settings.SPACE_EFFICIENCY_THICK) or \ - (parameter_space_efficiency and parameter_space_efficiency == array_space_efficiency) + (parameter_space_efficiency and parameter_space_efficiency == array_space_efficiency) def build_create_volume_in_volume_group_kwargs(pool, io_group, source_id): @@ -209,7 +210,7 @@ def _get_cli_volume_space_efficiency_aliases(cli_volume): return space_efficiency_aliases -class SVCArrayMediator(ArrayMediatorAbstract, VolumeGroupInterface): +class SVCArrayMediator(ArrayMediatorAbstract, VolumeGroupInterface, FenceInterface): ARRAY_ACTIONS = {} BLOCK_SIZE_IN_BYTES = 512 MAX_LUN_NUMBER = 511 @@ -871,9 +872,12 @@ def _create_snapshot(self, target_volume_name, source_cli_volume, space_efficien self._rollback_create_snapshot(target_volume_name) raise ex + def _lsmdiskgrp(self, **kwargs): + return self.client.svcinfo.lsmdiskgrp(**kwargs) + def _get_pool_site(self, pool): filter_value = 'name={}'.format(pool) - cli_pool = self.client.svcinfo.lsmdiskgrp(filtervalue=filter_value).as_single_element + cli_pool = self._lsmdiskgrp(filtervalue=filter_value).as_single_element if cli_pool: return cli_pool.site_name raise array_errors.PoolDoesNotExist(pool, self.endpoint) @@ -2028,3 +2032,43 @@ def add_volume_to_volume_group(self, volume_group_id, volume_id): def remove_volume_from_volume_group(self, volume_id): cli_volume = self._get_cli_volume_by_wwn(volume_id, not_exist_err=True) self._change_volume_group(cli_volume.id, None) + + def _get_ownership_group_pools(self, ownership_group): + filter_value = 'owner_name={}'.format(ownership_group) + cli_pools = self._lsmdiskgrp(filtervalue=filter_value).as_list + return cli_pools + + def is_fenced(self, fence_ownership_group): + ownership_group_pools = self._get_ownership_group_pools(fence_ownership_group) + if len(ownership_group_pools) == 0: + return True + + return False + + def _chmdiskgrp(self, pool_id, **cli_kwargs): + self.client.svctask.chmdiskgrp(object_id=pool_id, **cli_kwargs) + + def fence(self, fence_ownership_group, unfence_ownership_group): + ownership_group_pools = self._get_ownership_group_pools(fence_ownership_group) + if len(ownership_group_pools) == 0: + return + + self._remove_all_mappings_from_ownership_group(fence_ownership_group) + + for pool in ownership_group_pools: + self._chmdiskgrp(pool.id, ownershipgroup=unfence_ownership_group) + + def _remove_all_mappings_from_ownership_group(self, ownership_group): + filter_value = 'owner_name={}'.format(ownership_group) + + hosts = self.client.svcinfo.lshost(filtervalue=filter_value).as_list + host_names = [host.name for host in hosts] + + volumes = self._lsvdisk_list(filtervalue=filter_value) + volume_names = [volume.name for volume in volumes] + + mappings = self.client.svcinfo.lshostvdiskmap().as_list + + for mapping in mappings: + if mapping.name in host_names and mapping.vdisk_name in volume_names: + self.client.svctask.rmvdiskhostmap(vdisk_name=mapping.vdisk_name, host=mapping.name) diff --git a/controllers/array_action/fence_interface.py b/controllers/array_action/fence_interface.py new file mode 100644 index 000000000..ac2a797b2 --- /dev/null +++ b/controllers/array_action/fence_interface.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod + + +class FenceInterface(ABC): + + @abstractmethod + def is_fenced(self, fence_ownership_group): + """ + This function should check if the fence_ownership_group is already fenced (no pools in the og) + + Args: + fence_ownership_group : name of the ownership group that should be fenced + + Returns: + bool + + Raises: + None + """ + raise NotImplementedError + + @abstractmethod + def fence(self, fence_ownership_group, unfence_ownership_group): + """ + This function should fence the fence_ownership_group and unfence the unfence_ownership_group + + Args: + fence_ownership_group : name of the ownership group that should be fenced + unfence_ownership_group : name of the ownership group that should be unfenced + + Returns: + None + + Raises: + None + """ + raise NotImplementedError diff --git a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py index affc23823..84e015b99 100644 --- a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py +++ b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py @@ -3,7 +3,7 @@ from controllers.array_action.storage_agent import get_agent, detect_array_type from controllers.common.csi_logger import get_stdout_logger from controllers.servers import utils -from controllers.servers.csi.decorators import csi_method +from controllers.servers.csi.decorators import csi_fence_method logger = get_stdout_logger() @@ -31,14 +31,18 @@ def handle_fencing(request): class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): - @csi_method(error_response_type=fence_pb2.FenceClusterNetworkResponse, lock_request_attribute="parameters") + @csi_fence_method(error_response_type=fence_pb2.FenceClusterNetworkResponse) def FenceClusterNetwork(self, request, context): logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) return handle_fencing(request) - @csi_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse, lock_request_attribute="parameters") + @csi_fence_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse) def UnfenceClusterNetwork(self, request, context): logger.debug("UnfenceClusterNetwork parameters : {}".format(request.parameters)) logger.debug("UnfenceClusterNetwork cidrs : {}".format(request.cidrs)) return handle_fencing(request) + + @csi_fence_method(error_response_type=fence_pb2.ListClusterFenceResponse) + def ListClusterFence(self, request, context): + raise NotImplementedError() diff --git a/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py b/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py index 637254e4e..cf797e109 100644 --- a/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py +++ b/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py @@ -28,7 +28,8 @@ def GetCapabilities(self, request, context): logger.info("GetCapabilities") response = pb2.GetCapabilitiesResponse( capabilities=[self._get_replication_capability(), - self._get_controller_capability()]) + self._get_controller_capability(), + self._get_NetworkFence_capability()]) logger.info("finished GetCapabilities") return response @@ -45,6 +46,12 @@ def _get_controller_capability(self): return pb2.Capability( service=pb2.Capability.Service(type=capability_enum_value)) + def _get_NetworkFence_capability(self): + types = pb2.Capability.NetworkFence.Type + capability_enum_value = types.Value("NETWORK_FENCE") + return pb2.Capability( + network_fence=pb2.Capability.NetworkFence(type=capability_enum_value)) + def Probe(self, request, context): context.set_code(grpc.StatusCode.OK) return pb2.ProbeResponse() diff --git a/controllers/servers/csi/decorators.py b/controllers/servers/csi/decorators.py index 957724ebe..c55aa476e 100644 --- a/controllers/servers/csi/decorators.py +++ b/controllers/servers/csi/decorators.py @@ -15,13 +15,19 @@ def csi_method(error_response_type, lock_request_attribute=''): @decorator def call_csi_method(controller_method, servicer, request, context): lock_id = getattr(request, lock_request_attribute, None) - if isinstance(lock_id, dict): - lock_id = lock_id.get('fenceToken', '') return _set_sync_lock(lock_id, lock_request_attribute, error_response_type, controller_method, servicer, request, context) return call_csi_method +def csi_fence_method(error_response_type): + @decorator + def call_csi_method(controller_method, servicer, request, context): + lock_id = request.parameters.get('fenceToken', '') + return _set_sync_lock(lock_id, 'fenceToken', error_response_type, + controller_method, servicer, request, context) + + return call_csi_method def csi_replication_method(error_response_type): @decorator diff --git a/controllers/tests/array_action/svc/fence_svc_test.py b/controllers/tests/array_action/svc/fence_svc_test.py new file mode 100644 index 000000000..3f861e8e7 --- /dev/null +++ b/controllers/tests/array_action/svc/fence_svc_test.py @@ -0,0 +1,45 @@ +import unittest + +from mock import Mock +from munch import Munch + +from controllers.tests.array_action.svc.array_mediator_svc_test import TestArrayMediatorSVC +from controllers.tests.common.test_settings import HOST_NAME, VOLUME_NAME, POOL_ID, POOL_NAME, FENCE_OWNERSHIP_GROUP, \ + UNFENCE_OWNERSHIP_GROUP + + +class MyTestCase(TestArrayMediatorSVC): + def test_is_fenced_true(self): + self.svc.client.svcinfo.lsmdiskgrp.return_value = Mock(as_list=[]) + is_fenced = self.svc.is_fenced(FENCE_OWNERSHIP_GROUP) + self.assertTrue(is_fenced) + self.svc.client.svcinfo.lsmdiskgrp.assert_called_once_with( + filtervalue='owner_name={}'.format(FENCE_OWNERSHIP_GROUP)) + + def test_is_fenced_false(self): + self.svc.client.svcinfo.lsmdiskgrp.return_value = Mock(as_list=[Munch({"name": POOL_NAME})]) + is_fenced = self.svc.is_fenced(FENCE_OWNERSHIP_GROUP) + self.assertFalse(is_fenced) + self.svc.client.svcinfo.lsmdiskgrp.assert_called_once_with( + filtervalue='owner_name={}'.format(FENCE_OWNERSHIP_GROUP)) + + def test_fence_rmvdiskhostmap_called(self): + self.svc.client.svcinfo.lsmdiskgrp.return_value = Mock(as_list=[Munch({"name": POOL_NAME, "id": POOL_ID})]) + self.svc.client.svcinfo.lshost.return_value = Mock(as_list=[Munch({"name": HOST_NAME})]) + self.svc.client.svcinfo.lsvdisk.return_value = Mock(as_list=[Munch({"name": VOLUME_NAME})]) + self.svc.client.svcinfo.lshostvdiskmap.return_value = Mock( + as_list=[Munch({"name": HOST_NAME, "vdisk_name": VOLUME_NAME})]) + self.svc.fence(FENCE_OWNERSHIP_GROUP, UNFENCE_OWNERSHIP_GROUP) + self.svc.client.svctask.rmvdiskhostmap.assert_called_once_with(vdisk_name=VOLUME_NAME, host=HOST_NAME) + + def test_fence_rmvdiskhostmap_not_called(self): + self.svc.client.svcinfo.lsmdiskgrp.return_value = Mock(as_list=[Munch({"name": POOL_NAME, "id": POOL_ID})]) + self.svc.client.svcinfo.lshost.return_value = Mock(as_list=[Munch({"name": HOST_NAME})]) + self.svc.client.svcinfo.lsvdisk.return_value = Mock(as_list=[Munch({"name": VOLUME_NAME})]) + self.svc.client.svcinfo.lshostvdiskmap.return_value = Mock(as_list=[]) + self.svc.fence(FENCE_OWNERSHIP_GROUP, UNFENCE_OWNERSHIP_GROUP) + self.svc.client.svctask.rmvdiskhostmap.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/controllers/tests/common/test_settings.py b/controllers/tests/common/test_settings.py index 21a87535f..f18350ca9 100644 --- a/controllers/tests/common/test_settings.py +++ b/controllers/tests/common/test_settings.py @@ -60,3 +60,9 @@ REQUEST_VOLUME_GROUP_ID = ID_FORMAT.format(INTERNAL_VOLUME_GROUP_ID, VOLUME_GROUP_NAME) HOST_OBJECT_TYPE = "host" + +POOL_NAME = "pool_name" +POOL_ID = "pool_id" + +FENCE_OWNERSHIP_GROUP = "fence_ownership_group" +UNFENCE_OWNERSHIP_GROUP = "unfence_ownership_group" \ No newline at end of file From b457d454143b1ec41949f0c6749cf33f4a59037a Mon Sep 17 00:00:00 2001 From: ArbelNathan Date: Wed, 14 Jun 2023 16:46:51 +0300 Subject: [PATCH 5/7] logger, consts and fixes Signed-off-by: ArbelNathan --- .../array_action/array_mediator_svc.py | 39 ++++++++++++------- controllers/array_action/svc_messages.py | 6 +++ .../fence_controller_servicer.py | 6 +-- .../identity_controller_servicer.py | 4 +- controllers/tests/common/test_settings.py | 3 +- .../identity_controller_servicer_test.py | 2 +- 6 files changed, 38 insertions(+), 22 deletions(-) diff --git a/controllers/array_action/array_mediator_svc.py b/controllers/array_action/array_mediator_svc.py index 482fdc59d..7c1557b2a 100644 --- a/controllers/array_action/array_mediator_svc.py +++ b/controllers/array_action/array_mediator_svc.py @@ -10,8 +10,8 @@ import controllers.array_action.errors as array_errors import controllers.array_action.settings as array_settings -from controllers.array_action import svc_messages import controllers.servers.settings as controller_settings +from controllers.array_action import svc_messages from controllers.array_action.array_action_types import Volume, Snapshot, Replication, Host, VolumeGroup, ThinVolume from controllers.array_action.array_mediator_abstract import ArrayMediatorAbstract from controllers.array_action.fence_interface import FenceInterface @@ -2034,6 +2034,7 @@ def remove_volume_from_volume_group(self, volume_id): self._change_volume_group(cli_volume.id, None) def _get_ownership_group_pools(self, ownership_group): + logger.info(svc_messages.GET_OWNERSHIP_GROUP_POOLS.format(ownership_group)) filter_value = 'owner_name={}'.format(ownership_group) cli_pools = self._lsmdiskgrp(filtervalue=filter_value).as_list return cli_pools @@ -2041,24 +2042,17 @@ def _get_ownership_group_pools(self, ownership_group): def is_fenced(self, fence_ownership_group): ownership_group_pools = self._get_ownership_group_pools(fence_ownership_group) if len(ownership_group_pools) == 0: + logger.info(svc_messages.NO_POOLS_FOUND_IN_OWNERSHIP_GROUP.format(fence_ownership_group)) return True + logger.info(svc_messages.POOLS_FOUND_IN_OWNERSHIP_GROUP.format(fence_ownership_group, ownership_group_pools)) return False def _chmdiskgrp(self, pool_id, **cli_kwargs): self.client.svctask.chmdiskgrp(object_id=pool_id, **cli_kwargs) - def fence(self, fence_ownership_group, unfence_ownership_group): - ownership_group_pools = self._get_ownership_group_pools(fence_ownership_group) - if len(ownership_group_pools) == 0: - return - - self._remove_all_mappings_from_ownership_group(fence_ownership_group) - - for pool in ownership_group_pools: - self._chmdiskgrp(pool.id, ownershipgroup=unfence_ownership_group) - def _remove_all_mappings_from_ownership_group(self, ownership_group): + logger.info(svc_messages.REMOVING_ALL_MAPPINGS_FROM_OWNERSHIP_GROUP.format(ownership_group)) filter_value = 'owner_name={}'.format(ownership_group) hosts = self.client.svcinfo.lshost(filtervalue=filter_value).as_list @@ -2069,6 +2063,23 @@ def _remove_all_mappings_from_ownership_group(self, ownership_group): mappings = self.client.svcinfo.lshostvdiskmap().as_list - for mapping in mappings: - if mapping.name in host_names and mapping.vdisk_name in volume_names: - self.client.svctask.rmvdiskhostmap(vdisk_name=mapping.vdisk_name, host=mapping.name) + relevant_mappings = [mapping for mapping in mappings if + mapping.name in host_names and mapping.vdisk_name in volume_names] + logger.info(svc_messages.REMOVING_MAPPINGS.format(relevant_mappings)) + for mapping in relevant_mappings: + self.client.svctask.rmvdiskhostmap(vdisk_name=mapping.vdisk_name, host=mapping.name) + + def _change_pools_ownership_group(self, ownership_group, pools): + logger.info(svc_messages.CHANGE_POOLS_OWNERSHIP_GROUP.format(ownership_group)) + for pool in pools: + self._chmdiskgrp(pool.id, ownershipgroup=ownership_group) + + def fence(self, fence_ownership_group, unfence_ownership_group): + ownership_group_pools = self._get_ownership_group_pools(fence_ownership_group) + if len(ownership_group_pools) == 0: + logger.info(svc_messages.NO_POOLS_FOUND_IN_OWNERSHIP_GROUP.format(fence_ownership_group)) + return + + self._remove_all_mappings_from_ownership_group(fence_ownership_group) + + self._change_pools_ownership_group(unfence_ownership_group, ownership_group_pools) diff --git a/controllers/array_action/svc_messages.py b/controllers/array_action/svc_messages.py index 9e84e4c13..f2ae29c82 100644 --- a/controllers/array_action/svc_messages.py +++ b/controllers/array_action/svc_messages.py @@ -10,3 +10,9 @@ CREATE_HOST_WITHOUT_IO_GROUP = 'Created host {} with port {}' CREATE_HOST_WITH_IO_GROUP = 'Created host {} with port [{}] and with io_group [{}]' CHANGE_HOST_PROTOCOL = 'Changed host {} protocol to: {}' +GET_OWNERSHIP_GROUP_POOLS = 'Getting pools for ownership group {}' +NO_POOLS_FOUND_IN_OWNERSHIP_GROUP = 'No pools found in ownership group {}' +POOLS_FOUND_IN_OWNERSHIP_GROUP = 'Pools found in ownership group {}: {}' +REMOVING_ALL_MAPPINGS_FROM_OWNERSHIP_GROUP = 'Removing all mappings from ownership group {}' +REMOVING_MAPPINGS = 'Removing mappings {}' +CHANGE_POOLS_OWNERSHIP_GROUP = 'Changing pools ownership group to {}' diff --git a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py index 84e015b99..2b6de1135 100644 --- a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py +++ b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py @@ -13,6 +13,7 @@ def _is_already_handled(mediator, fence_ownership_group): def _fence_cluster_network(mediator, fence_ownership_group, unfence_ownership_group): + logger.info("fencing {}".format(fence_ownership_group)) mediator.fence(fence_ownership_group, unfence_ownership_group) @@ -25,6 +26,7 @@ def handle_fencing(request): with get_agent(connection_info, array_type).get_mediator() as mediator: # idempotence - check if the fence_ownership_group is already fenced (no pools in the og) if _is_already_handled(mediator, fence_ownership_group): + logger.info("{} is fenced".format(fence_ownership_group)) return fence_pb2.FenceClusterNetworkResponse() _fence_cluster_network(mediator, fence_ownership_group, unfence_ownership_group) return fence_pb2.FenceClusterNetworkResponse() @@ -33,14 +35,10 @@ def handle_fencing(request): class FenceControllerServicer(fence_pb2_grpc.FenceControllerServicer): @csi_fence_method(error_response_type=fence_pb2.FenceClusterNetworkResponse) def FenceClusterNetwork(self, request, context): - logger.debug("FenceClusterNetwork parameters : {}".format(request.parameters)) - logger.debug("FenceClusterNetwork cidrs : {}".format(request.cidrs)) return handle_fencing(request) @csi_fence_method(error_response_type=fence_pb2.UnfenceClusterNetworkResponse) def UnfenceClusterNetwork(self, request, context): - logger.debug("UnfenceClusterNetwork parameters : {}".format(request.parameters)) - logger.debug("UnfenceClusterNetwork cidrs : {}".format(request.cidrs)) return handle_fencing(request) @csi_fence_method(error_response_type=fence_pb2.ListClusterFenceResponse) diff --git a/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py b/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py index cf797e109..460ffd943 100644 --- a/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py +++ b/controllers/servers/csi/csi_addons_server/identity_controller_servicer.py @@ -29,7 +29,7 @@ def GetCapabilities(self, request, context): response = pb2.GetCapabilitiesResponse( capabilities=[self._get_replication_capability(), self._get_controller_capability(), - self._get_NetworkFence_capability()]) + self._get_network_fence_capability()]) logger.info("finished GetCapabilities") return response @@ -46,7 +46,7 @@ def _get_controller_capability(self): return pb2.Capability( service=pb2.Capability.Service(type=capability_enum_value)) - def _get_NetworkFence_capability(self): + def _get_network_fence_capability(self): types = pb2.Capability.NetworkFence.Type capability_enum_value = types.Value("NETWORK_FENCE") return pb2.Capability( diff --git a/controllers/tests/common/test_settings.py b/controllers/tests/common/test_settings.py index f18350ca9..eac674cfb 100644 --- a/controllers/tests/common/test_settings.py +++ b/controllers/tests/common/test_settings.py @@ -1,4 +1,5 @@ from controllers.common import settings as common_settings + SECRET_USERNAME_KEY = "username" SECRET_USERNAME_VALUE = "dummy_username" SECRET_PASSWORD_KEY = "password" @@ -65,4 +66,4 @@ POOL_ID = "pool_id" FENCE_OWNERSHIP_GROUP = "fence_ownership_group" -UNFENCE_OWNERSHIP_GROUP = "unfence_ownership_group" \ No newline at end of file +UNFENCE_OWNERSHIP_GROUP = "unfence_ownership_group" diff --git a/controllers/tests/controller_server/csi_addons/identity_controller_servicer_test.py b/controllers/tests/controller_server/csi_addons/identity_controller_servicer_test.py index c938d60b3..9e3aeaf8a 100644 --- a/controllers/tests/controller_server/csi_addons/identity_controller_servicer_test.py +++ b/controllers/tests/controller_server/csi_addons/identity_controller_servicer_test.py @@ -51,7 +51,7 @@ def test_get_identity_fails_when_name_or_version_are_empty(self, identity_config def test_get_capabilities_succeeds(self): response = self.servicer.GetCapabilities(self.request, self.context) - supported_capabilities = 2 + supported_capabilities = 3 self.assertIn('VolumeReplication', dir(response.capabilities[0])) self.assertIn('Service', dir(response.capabilities[1])) self.assertEqual(len(response.capabilities), supported_capabilities) From 0f87272ea0aceaa784e2e340710cd5c98b312c0a Mon Sep 17 00:00:00 2001 From: arbenathan Date: Wed, 21 Jun 2023 18:11:06 +0300 Subject: [PATCH 6/7] format cleanup Signed-off-by: arbenathan --- controllers/array_action/array_mediator_svc.py | 10 +++++----- controllers/servers/csi/decorators.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/controllers/array_action/array_mediator_svc.py b/controllers/array_action/array_mediator_svc.py index 5e118fe38..47d7cb6a3 100644 --- a/controllers/array_action/array_mediator_svc.py +++ b/controllers/array_action/array_mediator_svc.py @@ -1,7 +1,7 @@ from collections import defaultdict +from datetime import datetime, timedelta from io import StringIO from random import choice -from datetime import datetime, timedelta from packaging.version import Version from pysvc import errors as svc_errors @@ -9,22 +9,22 @@ from pysvc.unified.response import CLIFailureError, SVCResponse from retry import retry -from controllers.common.config import config import controllers.array_action.errors as array_errors import controllers.array_action.settings as array_settings -from controllers.array_action.registration_cache import SVC_REGISTRATION_CACHE import controllers.servers.settings as controller_settings from controllers.array_action import svc_messages -from controllers.servers.csi.decorators import register_csi_plugin from controllers.array_action.array_action_types import Volume, Snapshot, Replication, Host, VolumeGroup, ThinVolume from controllers.array_action.array_mediator_abstract import ArrayMediatorAbstract from controllers.array_action.fence_interface import FenceInterface +from controllers.array_action.registration_cache import SVC_REGISTRATION_CACHE from controllers.array_action.utils import ClassProperty, convert_scsi_id_to_nguid from controllers.array_action.volume_group_interface import VolumeGroupInterface from controllers.common import settings as common_settings +from controllers.common.config import config from controllers.common.csi_logger import get_stdout_logger -from controllers.servers.utils import get_connectivity_type_ports, split_string, is_call_home_enabled +from controllers.servers.csi.decorators import register_csi_plugin from controllers.servers.settings import UNIQUE_KEY_KEY +from controllers.servers.utils import get_connectivity_type_ports, split_string, is_call_home_enabled array_connections_dict = {} logger = get_stdout_logger() diff --git a/controllers/servers/csi/decorators.py b/controllers/servers/csi/decorators.py index b247a8dfc..938f6a2af 100644 --- a/controllers/servers/csi/decorators.py +++ b/controllers/servers/csi/decorators.py @@ -23,6 +23,7 @@ def call_csi_method(controller_method, servicer, request, context): return call_csi_method + def csi_fence_method(error_response_type): @decorator def call_csi_method(controller_method, servicer, request, context): @@ -32,6 +33,7 @@ def call_csi_method(controller_method, servicer, request, context): return call_csi_method + def csi_replication_method(error_response_type): @decorator def call_csi_method(controller_method, servicer, request, context): From 5e778b3c7a8b354ef9e2920979ac9351a7ca441b Mon Sep 17 00:00:00 2001 From: arbenathan Date: Sun, 25 Jun 2023 10:39:00 +0300 Subject: [PATCH 7/7] add validation for parameters Signed-off-by: arbenathan --- .../fence_controller_servicer.py | 1 + controllers/servers/utils.py | 20 ++++++++++++++++--- .../fence_controller_servicer_test.py | 12 +++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py index 2b6de1135..2925f8e45 100644 --- a/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py +++ b/controllers/servers/csi/csi_addons_server/fence_controller_servicer.py @@ -18,6 +18,7 @@ def _fence_cluster_network(mediator, fence_ownership_group, unfence_ownership_gr def handle_fencing(request): + utils.validate_fencing_request(request) fence_ownership_group = request.parameters["fenceToken"] unfence_ownership_group = request.parameters["unfenceToken"] diff --git a/controllers/servers/utils.py b/controllers/servers/utils.py index 2cc7b7ddf..19de76421 100644 --- a/controllers/servers/utils.py +++ b/controllers/servers/utils.py @@ -1,8 +1,8 @@ -from os import getenv import json import re from hashlib import sha256 from operator import eq +from os import getenv import base58 from csi_general import csi_pb2, volumegroup_pb2 @@ -278,7 +278,7 @@ def _validate_object_id(object_id, object_type=servers_settings.VOLUME_TYPE_NAME raise ValidationException(messages.WRONG_FORMAT_MESSAGE.format("volume id")) -def _validate_request_required_field(field_value, field_name): +def _validate_required_field(field_value, field_name): logger.debug("validating request {}".format(field_name)) if not field_value: raise ValidationException(messages.PARAMETER_SHOULD_NOT_BE_EMPTY_MESSAGE.format(field_name)) @@ -286,7 +286,7 @@ def _validate_request_required_field(field_value, field_name): def _validate_minimum_request_fields(request, required_field_names): for required_field_name in required_field_names: - _validate_request_required_field(getattr(request, required_field_name), required_field_name) + _validate_required_field(getattr(request, required_field_name), required_field_name) validate_secrets(request.secrets) @@ -851,3 +851,17 @@ def get_replication_object_type_and_id_info(request): def is_call_home_enabled(): return getenv(servers_settings.ENABLE_CALL_HOME_ENV_VAR, 'true') == 'true' + + +def _validate_parameters_fields(parameters, required_parameters_names): + for required_field_name in required_parameters_names: + _validate_required_field(parameters.get(required_field_name), required_field_name) + + +def validate_fencing_request(request): + logger.debug("validating fencing request") + + _validate_parameters_fields(request.parameters, ["fenceToken", "unfenceToken"]) + validate_secrets(request.secrets) + + logger.debug("fencing validation finished") diff --git a/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py b/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py index 1e869abc4..05591ab95 100644 --- a/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py +++ b/controllers/tests/controller_server/csi_addons/fence_controller_servicer_test.py @@ -40,6 +40,18 @@ def test_fence_fails(self): self.assertEqual(grpc.StatusCode.INTERNAL, self.context.code) self.mediator.fence.assert_called_once_with("fenceToken", "unfenceToken") + def test_fence_invalid_parameters(self): + self.request.parameters = {} + self.servicer.FenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.INVALID_ARGUMENT, self.context.code) + self.mediator.fence.assert_not_called() + + def test_fence_invalid_secret(self): + self.request.secrets = {} + self.servicer.FenceClusterNetwork(self.request, self.context) + self.assertEqual(grpc.StatusCode.INVALID_ARGUMENT, self.context.code) + self.mediator.fence.assert_not_called() + def test_fence_already_fenced(self): self.mediator.is_fenced.return_value = True self.servicer.FenceClusterNetwork(self.request, self.context)