From 892813de1dac672b50a3839960251aa0ef607442 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 18 Jan 2024 20:54:19 -0500 Subject: [PATCH 01/28] add isilon plugin --- coldfront/config/plugins/isilon.py | 8 ++++++++ coldfront/config/settings.py | 19 ++++++++++--------- coldfront/plugins/isilon/__init__.py | 0 coldfront/plugins/isilon/utils.py | 15 +++++++++++++++ requirements.txt | 1 + 5 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 coldfront/config/plugins/isilon.py create mode 100644 coldfront/plugins/isilon/__init__.py create mode 100644 coldfront/plugins/isilon/utils.py diff --git a/coldfront/config/plugins/isilon.py b/coldfront/config/plugins/isilon.py new file mode 100644 index 000000000..91c85f00a --- /dev/null +++ b/coldfront/config/plugins/isilon.py @@ -0,0 +1,8 @@ +from coldfront.config.env import ENV +from coldfront.config.logging import LOGGING +from coldfront.config.base import INSTALLED_APPS + +INSTALLED_APPS += [ 'coldfront.plugins.isilon' ] +ISILON_USER = ENV.str('ISILON_USER') +ISILON_PASS = ENV.str('ISILON_PASS') + diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index f6d980eb9..05ceeaaef 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -18,20 +18,21 @@ # ColdFront plugin settings plugin_configs = { - 'PLUGIN_SLURM': 'plugins/slurm.py', - 'PLUGIN_IQUOTA': 'plugins/iquota.py', - 'PLUGIN_FREEIPA': 'plugins/freeipa.py', - 'PLUGIN_SYSMON': 'plugins/system_montior.py', - 'PLUGIN_XDMOD': 'plugins/xdmod.py', + 'PLUGIN_API': 'plugins/api.py', 'PLUGIN_AUTH_OIDC': 'plugins/openid.py', 'PLUGIN_AUTH_LDAP': 'plugins/ldap.py', - 'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py', - 'PLUGIN_API': 'plugins/api.py', 'PLUGIN_LDAP': 'plugins/ldap_fasrc.py', - 'PLUGIN_SFTOCF': 'plugins/sftocf.py', + 'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py', 'PLUGIN_FASRC': 'plugins/fasrc.py', - 'PLUGIN_IFX': 'plugins/ifx.py', 'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py', + 'PLUGIN_FREEIPA': 'plugins/freeipa.py', + 'PLUGIN_IFX': 'plugins/ifx.py', + 'PLUGIN_ISILON': 'plugins/isilon.py', + 'PLUGIN_IQUOTA': 'plugins/iquota.py', + 'PLUGIN_SFTOCF': 'plugins/sftocf.py', + 'PLUGIN_SYSMON': 'plugins/system_montior.py', + 'PLUGIN_SLURM': 'plugins/slurm.py', + 'PLUGIN_XDMOD': 'plugins/xdmod.py', } # This allows plugins to be enabled via environment variables. Can alternatively diff --git a/coldfront/plugins/isilon/__init__.py b/coldfront/plugins/isilon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py new file mode 100644 index 000000000..8009c1c7f --- /dev/null +++ b/coldfront/plugins/isilon/utils.py @@ -0,0 +1,15 @@ +import isilon_sdk.v9_5_0 +from isilon_sdk.v9_5_0.rest import ApiException + +from coldfront.config.env import ENV +from coldfront.core.utils.common import import_from_settings + +def connect(): + configuration = isilon_sdk.v9_5_0.Configuration() + configuration.host = 'http://holy-isilon01.rc.fas.harvard.edu:8080' + configuration.username = import_from_settings('ISILON_USER') + configuration.password = import_from_settings('ISILON_PASS') + configuration.verify_ssl = False + api_client = isilon_sdk.v9_5_0.ApiClient(configuration) + api_instance = isilon_sdk.v9_5_0.ProtocolsApi(api_client) + return api_instance diff --git a/requirements.txt b/requirements.txt index b5c084c34..55fa5334a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ factory-boy==3.2.1 future==0.18.3 humanize==4.6.0 idna==3.4 +isilon-sdk pyparsing==3.0.9 python-dateutil==2.8.2 python-memcached==1.59 From c7a6210e03e35040fc3dd4e07e983ee2d8100d7c Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 23 Jan 2024 14:48:26 -0800 Subject: [PATCH 02/28] add basic quota update function --- coldfront/plugins/isilon/utils.py | 84 ++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 8009c1c7f..a36327574 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,15 +1,83 @@ -import isilon_sdk.v9_5_0 -from isilon_sdk.v9_5_0.rest import ApiException +import isilon_sdk.v9_3_0 +from isilon_sdk.v9_3_0.rest import ApiException +import requests from coldfront.config.env import ENV +from coldfront.core.allocation.models import Allocation, AllocationAttributeType +from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings -def connect(): - configuration = isilon_sdk.v9_5_0.Configuration() - configuration.host = 'http://holy-isilon01.rc.fas.harvard.edu:8080' + +def connect(cluster_name): + configuration = isilon_sdk.v9_3_0.Configuration() + configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' configuration.username = import_from_settings('ISILON_USER') configuration.password = import_from_settings('ISILON_PASS') configuration.verify_ssl = False - api_client = isilon_sdk.v9_5_0.ApiClient(configuration) - api_instance = isilon_sdk.v9_5_0.ProtocolsApi(api_client) - return api_instance + api_client = isilon_sdk.v9_3_0.ApiClient(configuration) + return api_client + + +def update_quota_and_usage(alloc, usage_attribute_type, value_list): + usage_attribute, _ = alloc.allocation.allocationattribute_set.get_or_create( + allocation_attribute_type=usage_attribute_type + ) + usage_attribute.value = value_list[0] + usage_attribute.save() + usage = usage_attribute.allocationattributeusage + usage.value = value_list[1] + usage.save() + return usage_attribute + +def update_quotas_usages(): + """For all active tier1 allocations, update quota and usage + 1. run a query that collects all active tier1 allocations + """ + quota_bytes_attributetype = AllocationAttributeType.objects.get( + name='Quota_In_Bytes') + quota_tbs_attributetype = AllocationAttributeType.objects.get( + name='Storage Quota (TB)') + # create isilon connections to all isilon clusters in coldfront + isilon_resources = Resource.objects.filter(name__contains='tier1') + isilon_clusters = {} + for resource in isilon_resources: + resource_name = resource.name.split('/')[0] + # try connecting to the cluster. If it fails, display an error and + # replace the resource with a dummy resource + try: + api_client = connect(resource_name) + isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) + except Exception as e: + print(f'Could not connect to {resource_name} - will not update quotas for allocations on this resource') + print(e) + # isilon_clusters[resource.name] = None + + isilon_allocations = Allocation.objects.filter( + is_active=True, + resource__in=isilon_clusters.keys(), + ) + + for allocation in isilon_allocations: + # get the api instance for this allocation. If it doesn't exist, skip + api_instance = isilon_clusters[allocation.resource.name] + if not api_instance: + continue + try: + api_response = api_instance.get_quota_quota( + path=allocation.resource.path, + recurse_path_children=True, + ) + except ApiException as e: + print("Exception when calling QuotaApi->list_quotas: %s\n" % e) + continue + # update the quota and usage for this allocation + quota = api_response['thresholds']['hard'] + usage = api_response['usage']['fslogical'] + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) From 7e3a288f05dd09d7ff4b6fa662d9567dd7a0c2c9 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Fri, 26 Jan 2024 12:51:58 -0800 Subject: [PATCH 03/28] add command --- .../management/commands/pull_isilon_quotas.py | 12 ++++++++++++ coldfront/plugins/isilon/utils.py | 14 +++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py diff --git a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py new file mode 100644 index 000000000..0d9f42961 --- /dev/null +++ b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from coldfront.plugins.isilon.utils import update_quotas_usages + +class Command(BaseCommand): + """ + Pull Isilon quotas + """ + help = 'Pull Isilon quotas' + + def handle(self, *args, **kwargs): + update_quotas_usages() diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index a36327574..71004b0dd 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,12 +1,13 @@ +import logging + import isilon_sdk.v9_3_0 from isilon_sdk.v9_3_0.rest import ApiException -import requests -from coldfront.config.env import ENV from coldfront.core.allocation.models import Allocation, AllocationAttributeType from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings +logger = logging.getLogger(__name__) def connect(cluster_name): configuration = isilon_sdk.v9_3_0.Configuration() @@ -48,8 +49,9 @@ def update_quotas_usages(): api_client = connect(resource_name) isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) except Exception as e: - print(f'Could not connect to {resource_name} - will not update quotas for allocations on this resource') - print(e) + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + logger.warning("%s Error: %s", message, e) + print(f"{message} Error: {e}") # isilon_clusters[resource.name] = None isilon_allocations = Allocation.objects.filter( @@ -68,7 +70,9 @@ def update_quotas_usages(): recurse_path_children=True, ) except ApiException as e: - print("Exception when calling QuotaApi->list_quotas: %s\n" % e) + message = f'Exception when calling QuotaApi->list_quotas: {e}' + print(message) + logger.warning(message) continue # update the quota and usage for this allocation quota = api_response['thresholds']['hard'] From 517c0993e8465e5e89c1337362e99cd65619887e Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 09:25:57 -0800 Subject: [PATCH 04/28] change structure of allocation collection --- coldfront/plugins/isilon/utils.py | 57 ++++++++++++++----------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 71004b0dd..48fae3323 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -40,48 +40,43 @@ def update_quotas_usages(): name='Storage Quota (TB)') # create isilon connections to all isilon clusters in coldfront isilon_resources = Resource.objects.filter(name__contains='tier1') - isilon_clusters = {} for resource in isilon_resources: resource_name = resource.name.split('/')[0] # try connecting to the cluster. If it fails, display an error and # replace the resource with a dummy resource try: api_client = connect(resource_name) - isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) + api_instance = isilon_sdk.v9_3_0.QuotaApi(api_client) except Exception as e: message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' logger.warning("%s Error: %s", message, e) print(f"{message} Error: {e}") # isilon_clusters[resource.name] = None - - isilon_allocations = Allocation.objects.filter( - is_active=True, - resource__in=isilon_clusters.keys(), - ) - - for allocation in isilon_allocations: - # get the api instance for this allocation. If it doesn't exist, skip - api_instance = isilon_clusters[allocation.resource.name] - if not api_instance: - continue - try: - api_response = api_instance.get_quota_quota( - path=allocation.resource.path, - recurse_path_children=True, - ) - except ApiException as e: - message = f'Exception when calling QuotaApi->list_quotas: {e}' - print(message) - logger.warning(message) continue - # update the quota and usage for this allocation - quota = api_response['thresholds']['hard'] - usage = api_response['usage']['fslogical'] - quota_tb = quota / 1024 / 1024 / 1024 / 1024 - usage_tb = usage / 1024 / 1024 / 1024 / 1024 - update_quota_and_usage( - allocation, quota_bytes_attributetype, [quota, usage] + + # get all active allocations for this resource + isilon_allocations = Allocation.objects.filter( + status__name='Active', + resources__name=resource.name, ) - update_quota_and_usage( - allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + + # get all allocation quotas and usoges + api_response = api_instance.list_quota_quotas( + path='/ifs/rc_labs/', + recurse_path_children=True, ) + + for allocation in isilon_allocations: + # get the api_response entry for this allocation. If it doesn't exist, skip + api_entry = next(e for e in api_response.quota_quotas if e['path'] == f'/ifs/{allocation.path}') + # update the quota and usage for this allocation + quota = api_entry['thresholds']['hard'] + usage = api_entry['usage']['fslogical'] + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) From 48cc52a2d0b1caaaaee1e4b081d8fc29d3df1801 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 16:20:14 -0500 Subject: [PATCH 05/28] make pipeline operational --- coldfront/plugins/isilon/utils.py | 46 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 48fae3323..9cca97cfb 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -20,7 +20,7 @@ def connect(cluster_name): def update_quota_and_usage(alloc, usage_attribute_type, value_list): - usage_attribute, _ = alloc.allocation.allocationattribute_set.get_or_create( + usage_attribute, _ = alloc.allocationattribute_set.get_or_create( allocation_attribute_type=usage_attribute_type ) usage_attribute.value = value_list[0] @@ -41,6 +41,7 @@ def update_quotas_usages(): # create isilon connections to all isilon clusters in coldfront isilon_resources = Resource.objects.filter(name__contains='tier1') for resource in isilon_resources: + report = {"complete": 0, "no entry": [], "empty quota": []} resource_name = resource.name.split('/')[0] # try connecting to the cluster. If it fails, display an error and # replace the resource with a dummy resource @@ -61,17 +62,40 @@ def update_quotas_usages(): ) # get all allocation quotas and usoges - api_response = api_instance.list_quota_quotas( - path='/ifs/rc_labs/', - recurse_path_children=True, - ) - + try: + rc_labs = api_instance.list_quota_quotas( + path='/ifs/rc_labs/', + recurse_path_children=True, + ) + l3_labs = api_instance.list_quota_quotas( + path='/ifs/rc_fasse_labs/', + recurse_path_children=True, + ) + except Exception as e: + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + logger.warning("%s Error: %s", message, e) + print(f"{message} Error: {e}") + # isilon_clusters[resource.name] = None + continue + quotas = rc_labs.quotas + l3_labs.quotas for allocation in isilon_allocations: # get the api_response entry for this allocation. If it doesn't exist, skip - api_entry = next(e for e in api_response.quota_quotas if e['path'] == f'/ifs/{allocation.path}') + + try: + api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') + except StopIteration as e: + logger.error('no isilon quota entry for allocation %s', allocation) + print('no isilon quota entry for allocation', allocation) + report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue # update the quota and usage for this allocation - quota = api_entry['thresholds']['hard'] - usage = api_entry['usage']['fslogical'] + quota = api_entry.thresholds.hard + usage = api_entry.usage.fslogical + if quota is None: + logger.error('no hard threshold set for allocation %s', allocation) + print('no hard threshold set for allocation', allocation) + report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue quota_tb = quota / 1024 / 1024 / 1024 / 1024 usage_tb = usage / 1024 / 1024 / 1024 / 1024 update_quota_and_usage( @@ -80,3 +104,7 @@ def update_quotas_usages(): update_quota_and_usage( allocation, quota_tbs_attributetype, [quota_tb, usage_tb] ) + print("SUCCESS:update for allocation", allocation, "complete") + report['complete'] += 1 + print(report) + logger.warning("isilon update report for %s: %s", resource_name, report) From 213308b95ab8e43f1cd1f731bb9efee2a5f05d94 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 13:24:02 -0800 Subject: [PATCH 06/28] add djangoq task --- coldfront/plugins/isilon/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 coldfront/plugins/isilon/tasks.py diff --git a/coldfront/plugins/isilon/tasks.py b/coldfront/plugins/isilon/tasks.py new file mode 100644 index 000000000..8796ae153 --- /dev/null +++ b/coldfront/plugins/isilon/tasks.py @@ -0,0 +1,6 @@ +from coldfront.plugins.isilon.utils import update_quotas_usages + +def pull_isilon_quotas(): + """Pull Isilon quotas + """ + update_quotas_usages() From d8b398448f693807ac5cd844de0728a58139460d Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 30 Jan 2024 10:14:41 -0500 Subject: [PATCH 07/28] config fixes --- coldfront/config/plugins/isilon.py | 4 ++-- coldfront/config/settings.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/config/plugins/isilon.py b/coldfront/config/plugins/isilon.py index 91c85f00a..6b1169bc0 100644 --- a/coldfront/config/plugins/isilon.py +++ b/coldfront/config/plugins/isilon.py @@ -3,6 +3,6 @@ from coldfront.config.base import INSTALLED_APPS INSTALLED_APPS += [ 'coldfront.plugins.isilon' ] -ISILON_USER = ENV.str('ISILON_USER') -ISILON_PASS = ENV.str('ISILON_PASS') +ISILON_USER = ENV.str('ISILON_USER', '') +ISILON_PASS = ENV.str('ISILON_PASS', '') diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 05ceeaaef..725217259 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -30,7 +30,7 @@ 'PLUGIN_ISILON': 'plugins/isilon.py', 'PLUGIN_IQUOTA': 'plugins/iquota.py', 'PLUGIN_SFTOCF': 'plugins/sftocf.py', - 'PLUGIN_SYSMON': 'plugins/system_montior.py', + 'PLUGIN_SYSMON': 'plugins/system_monitor.py', 'PLUGIN_SLURM': 'plugins/slurm.py', 'PLUGIN_XDMOD': 'plugins/xdmod.py', } From ae311f5448030a42c36d7f14fd40cb10b3560a8d Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 30 Jan 2024 10:27:05 -0500 Subject: [PATCH 08/28] allocation admin changes --- coldfront/core/allocation/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/allocation/admin.py b/coldfront/core/allocation/admin.py index da2509250..ace54c28e 100644 --- a/coldfront/core/allocation/admin.py +++ b/coldfront/core/allocation/admin.py @@ -29,7 +29,7 @@ class AllocationStatusChoiceAdmin(admin.ModelAdmin): class AllocationUserInline(admin.TabularInline): model = AllocationUser extra = 0 - fields = ('id', 'user', 'status', 'usage',) + fields = ('id', 'user', 'status', 'usage', 'usage_bytes', 'unit') class AllocationAttributeInline(admin.TabularInline): From d4d3a0c2cb9c2e8977238ac06d738e8ce9649811 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 12:17:00 -0800 Subject: [PATCH 09/28] restructure isilon utils --- .../management/commands/pull_isilon_quotas.py | 84 ++++++++- coldfront/plugins/isilon/utils.py | 161 ++++++++---------- 2 files changed, 153 insertions(+), 92 deletions(-) diff --git a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py index 0d9f42961..33e052df0 100644 --- a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py +++ b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py @@ -1,6 +1,16 @@ +import logging + from django.core.management.base import BaseCommand -from coldfront.plugins.isilon.utils import update_quotas_usages +from coldfront.core.allocation.models import Allocation, AllocationAttributeType +from coldfront.core.resource.models import Resource +from coldfront.plugins.isilon.utils import ( + IsilonConnection, + print_log_error, + update_coldfront_quota_and_usage, +) + +logger = logging.getLogger(__name__) class Command(BaseCommand): """ @@ -9,4 +19,74 @@ class Command(BaseCommand): help = 'Pull Isilon quotas' def handle(self, *args, **kwargs): - update_quotas_usages() + """For all active tier1 allocations, update quota and usage + 1. run a query that collects all active tier1 allocations + """ + quota_bytes_attributetype = AllocationAttributeType.objects.get( + name='Quota_In_Bytes') + quota_tbs_attributetype = AllocationAttributeType.objects.get( + name='Storage Quota (TB)') + # create isilon connections to all isilon clusters in coldfront + isilon_resources = Resource.objects.filter(name__contains='tier1') + for resource in isilon_resources: + report = {"complete": 0, "no entry": [], "empty quota": []} + resource_name = resource.name.split('/')[0] + # try connecting to the cluster. If it fails, display an error and + # replace the resource with a dummy resource + try: + api_instance = IsilonConnection(resource_name) + except Exception as e: + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + print_log_error(e, message) + # isilon_clusters[resource.name] = None + continue + + # get all active allocations for this resource + isilon_allocations = Allocation.objects.filter( + status__name='Active', + resources__name=resource.name, + ) + + # get all allocation quotas and usoges + try: + rc_labs = api_instance.quota_client.list_quota_quotas( + path='/ifs/rc_labs/', recurse_path_children=True, + ) + l3_labs = api_instance.quota_client.list_quota_quotas( + path='/ifs/rc_fasse_labs/', recurse_path_children=True, + ) + except Exception as e: + err = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + print_log_error(e, err) + # isilon_clusters[resource.name] = None + continue + quotas = rc_labs.quotas + l3_labs.quotas + for allocation in isilon_allocations: + # get the api_response entry for this allocation. If it doesn't exist, skip + try: + api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') + except StopIteration as e: + err = f'no isilon quota entry for allocation {allocation}' + print_log_error(e, err) + report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue + # update the quota and usage for this allocation + quota = api_entry.thresholds.hard + usage = api_entry.usage.fslogical + if quota is None: + err = f'no hard threshold set for allocation {allocation}' + print_log_error(None, err) + report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_coldfront_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_coldfront_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) + print("SUCCESS:update for allocation", allocation, "complete") + report['complete'] += 1 + print(report) + logger.warning("isilon update report for %s: %s", resource_name, report) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 9cca97cfb..5f1438e42 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,25 +1,84 @@ import logging -import isilon_sdk.v9_3_0 +import isilon_sdk.v9_3_0 as isilon_api from isilon_sdk.v9_3_0.rest import ApiException -from coldfront.core.allocation.models import Allocation, AllocationAttributeType -from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings logger = logging.getLogger(__name__) -def connect(cluster_name): - configuration = isilon_sdk.v9_3_0.Configuration() - configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' - configuration.username = import_from_settings('ISILON_USER') - configuration.password = import_from_settings('ISILON_PASS') - configuration.verify_ssl = False - api_client = isilon_sdk.v9_3_0.ApiClient(configuration) - return api_client +class IsilonConnection: + """Convenience class containing methods for collecting data from an isilon cluster + """ + def __init__(self, cluster_name): + self.cluster_name = cluster_name + self.api_client = self.connect(cluster_name) + self.quota_client = isilon_api.QuotaApi(self.api_client) + self.pools_client = isilon_api.StoragepoolApi(self.api_client) + + def connect(self, cluster_name): + configuration = isilon_api.Configuration() + configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' + configuration.username = import_from_settings('ISILON_USER') + configuration.password = import_from_settings('ISILON_PASS') + configuration.verify_ssl = False + api_client = isilon_api.ApiClient(configuration) + return api_client + + def get_isilon_volume_unallocated_space(self): + """get the total unallocated space on a volume + calculated as total usable space minus sum of all quotas on the volume + """ + try: + quotas = self.quota_client.list_quota_quotas(type='directory') + quota_sum = sum( + [q.thresholds.hard for q in quotas.quotas if q.thresholds.hard]) + pool_query = self.pools_client.get_storagepool_storagepools( + toplevels=True) + total_space = pool_query.usage.usable_bytes + return total_space - quota_sum + except ApiException as e: + err = f'ERROR: could not get quota for {self.cluster_name} - {e}' + print_log_error(e, err) + return None + +def update_isilon_allocation_quota(allocation, quota): + """Update the quota for an allocation on an isilon cluster + + Parameters + ---------- + api_instance : isilon_api.QuotaApi + allocation : coldfront.core.allocation.models.Allocation + quota : int + """ + # make isilon connection to the allocation's resource + isilon_resource = allocation.resources.first().split('/')[0] + isilon_conn = IsilonConnection(isilon_resource) + path = f'/ifs/{allocation.path}' + + # check if enough space exists on the volume + quota_bytes = quota * 1024**3 + unallocated_space = isilon_conn.get_isilon_volume_unallocated_space() + allowable_space = unallocated_space * 0.8 + if allowable_space < quota_bytes: + raise ValueError( + 'ERROR: not enough space on volume to set quota to %s for %s' + % (quota, allocation) + ) + try: + isilon_conn.quota_client.update_quota_quota(path=path, threshold_hard=quota) + print(f"SUCCESS: updated quota for {allocation} to {quota}") + logger.info("SUCCESS: updated quota for %s to %s", allocation, quota) + except ApiException as e: + err = f"ERROR: could not update quota for {allocation} to {quota} - {e}" + print_log_error(e, err) -def update_quota_and_usage(alloc, usage_attribute_type, value_list): +def print_log_error(e, message): + print(f'ERROR: {message} - {e}') + logger.error("%s - %s", message, e) + +def update_coldfront_quota_and_usage(alloc, usage_attribute_type, value_list): usage_attribute, _ = alloc.allocationattribute_set.get_or_create( allocation_attribute_type=usage_attribute_type ) @@ -30,81 +89,3 @@ def update_quota_and_usage(alloc, usage_attribute_type, value_list): usage.save() return usage_attribute -def update_quotas_usages(): - """For all active tier1 allocations, update quota and usage - 1. run a query that collects all active tier1 allocations - """ - quota_bytes_attributetype = AllocationAttributeType.objects.get( - name='Quota_In_Bytes') - quota_tbs_attributetype = AllocationAttributeType.objects.get( - name='Storage Quota (TB)') - # create isilon connections to all isilon clusters in coldfront - isilon_resources = Resource.objects.filter(name__contains='tier1') - for resource in isilon_resources: - report = {"complete": 0, "no entry": [], "empty quota": []} - resource_name = resource.name.split('/')[0] - # try connecting to the cluster. If it fails, display an error and - # replace the resource with a dummy resource - try: - api_client = connect(resource_name) - api_instance = isilon_sdk.v9_3_0.QuotaApi(api_client) - except Exception as e: - message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - logger.warning("%s Error: %s", message, e) - print(f"{message} Error: {e}") - # isilon_clusters[resource.name] = None - continue - - # get all active allocations for this resource - isilon_allocations = Allocation.objects.filter( - status__name='Active', - resources__name=resource.name, - ) - - # get all allocation quotas and usoges - try: - rc_labs = api_instance.list_quota_quotas( - path='/ifs/rc_labs/', - recurse_path_children=True, - ) - l3_labs = api_instance.list_quota_quotas( - path='/ifs/rc_fasse_labs/', - recurse_path_children=True, - ) - except Exception as e: - message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - logger.warning("%s Error: %s", message, e) - print(f"{message} Error: {e}") - # isilon_clusters[resource.name] = None - continue - quotas = rc_labs.quotas + l3_labs.quotas - for allocation in isilon_allocations: - # get the api_response entry for this allocation. If it doesn't exist, skip - - try: - api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') - except StopIteration as e: - logger.error('no isilon quota entry for allocation %s', allocation) - print('no isilon quota entry for allocation', allocation) - report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - # update the quota and usage for this allocation - quota = api_entry.thresholds.hard - usage = api_entry.usage.fslogical - if quota is None: - logger.error('no hard threshold set for allocation %s', allocation) - print('no hard threshold set for allocation', allocation) - report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - quota_tb = quota / 1024 / 1024 / 1024 / 1024 - usage_tb = usage / 1024 / 1024 / 1024 / 1024 - update_quota_and_usage( - allocation, quota_bytes_attributetype, [quota, usage] - ) - update_quota_and_usage( - allocation, quota_tbs_attributetype, [quota_tb, usage_tb] - ) - print("SUCCESS:update for allocation", allocation, "complete") - report['complete'] += 1 - print(report) - logger.warning("isilon update report for %s: %s", resource_name, report) From ac432efd5c7f3f6c706712a4f1a900608a0b019c Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 16:37:56 -0800 Subject: [PATCH 10/28] attach isilon auto-update to GUI --- coldfront/core/allocation/forms.py | 25 +++++++- .../allocation/allocation_change_detail.html | 6 +- coldfront/core/allocation/views.py | 60 ++++++++++++++++++- coldfront/plugins/isilon/utils.py | 2 +- 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index 592785795..8de865baf 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -40,10 +40,10 @@ def validate(self, value): "Input must consist only of digits (or x'es) and dashes." ) if len(digits_only(value)) != 33: - raise ValidationError("Input must contain exactly 33 digits.") + raise ValidationError('Input must contain exactly 33 digits.') if 'x' in digits_only(value)[:8]+digits_only(value)[12:]: raise ValidationError( - "xes are only allowed in place of the product code (the third grouping of characters in the code)" + 'xes are only allowed in place of the product code (the third grouping of characters in the code)' ) def clean(self, value): @@ -123,7 +123,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): ).order_by("user__username").exclude(user=project_obj.pi) # if user_query_set: # self.fields['users'].choices = ((user.user.username, "%s %s (%s)" % ( - # user.user.first_name, user.user.last_name, user.user.username)) for user in user_query_set) + # u.user.first_name, u.user.last_name, u.user.username)) for u in user_query_set) # self.fields['users'].help_text = '
Select users in your project to add to this allocation.' # else: # self.fields['users'].widget = forms.HiddenInput() @@ -380,6 +380,25 @@ class AllocationChangeNoteForm(forms.Form): widget=forms.Textarea, help_text="Leave any feedback about the allocation change request.") + +ALLOCATION_AUTOUPDATE_OPTIONS = [ + ('1', 'I have already manually modified the allocation.'), + ('2', 'I would like Coldfront to modify the allocation for me. If Coldfront experiences any issues with the modification process, I understand that I will need to modify the allocation manually instead.'), +] + +class AllocationAutoUpdateForm(forms.Form): + sheetcheck = forms.BooleanField( + label='I have ensured that enough space is available on this resource.' + ) + auto_update_opts = forms.ChoiceField( + label='How will this allocation be modified?', + required=True, + widget=forms.RadioSelect, + choices=ALLOCATION_AUTOUPDATE_OPTIONS, + ) + + + class AllocationAttributeCreateForm(forms.ModelForm): class Meta: model = AllocationAttribute diff --git a/coldfront/core/allocation/templates/allocation/allocation_change_detail.html b/coldfront/core/allocation/templates/allocation/allocation_change_detail.html index 6d8fcd82d..0bf4e15ca 100644 --- a/coldfront/core/allocation/templates/allocation/allocation_change_detail.html +++ b/coldfront/core/allocation/templates/allocation/allocation_change_detail.html @@ -193,8 +193,9 @@

Actio
- - {% csrf_token %} + {% csrf_token %} + {{autoupdate_form.sheetcheck | as_crispy_field}} + {{autoupdate_form.auto_update_opts | as_crispy_field}} {{note_form.notes | as_crispy_field}}
{% if allocation_change.status.name == 'Pending' %} @@ -205,7 +206,6 @@

Actio Update

-
{% endif %} diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index 3be2aa744..f63d54123 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -35,6 +35,7 @@ AllocationAttributeChangeForm, AllocationAttributeUpdateForm, AllocationForm, + AllocationAutoUpdateForm, AllocationInvoiceNoteDeleteForm, AllocationInvoiceUpdateForm, AllocationRemoveUserForm, @@ -72,6 +73,8 @@ from ifxbilling.models import Account, UserProductAccount if 'django_q' in settings.INSTALLED_APPS: from django_q.tasks import Task +if 'coldfront.plugins.isilon' in settings.INSTALLED_APPS: + from coldfront.plugins.isilon import update_isilon_allocation_quota ALLOCATION_ENABLE_ALLOCATION_RENEWAL = import_from_settings( 'ALLOCATION_ENABLE_ALLOCATION_RENEWAL', True) @@ -1718,9 +1721,12 @@ def get(self, request, *args, **kwargs): initial={'notes': allocation_change_obj.notes} ) + autoupdate_form = AllocationAutoUpdateForm() + context = self.get_context_data() context['allocation_change_form'] = allocation_change_form + context['autoupdate_form'] = autoupdate_form context['note_form'] = note_form return render(request, self.template_name, context) @@ -1765,7 +1771,10 @@ def post(self, request, *args, **kwargs): context['allocation_change_form'] = allocation_change_form return render(request, self.template_name, context) + action = request.POST.get('action') + + autoupdate_form = AllocationAutoUpdateForm(request.POST) if action not in ['update', 'approve', 'deny']: return HttpResponseBadRequest('Invalid request') @@ -1845,9 +1854,54 @@ def post(self, request, *args, **kwargs): alloc_change_obj.allocation.save() if attrs_to_change: - attr_changes = ( - alloc_change_obj.allocationattributechangerequest_set.all() - ) + attr_changes = alloc_change_obj.allocationattributechangerequest_set.all() + + autoupdate_choice = autoupdate_form.data.get('auto_update_opts') + if autoupdate_choice == '2': + # check resource type, see if appropriate plugin is available + resource = alloc_change_obj.allocation.resources.first().name + resources_plugins = { + 'isilon': 'coldfront.plugins.isilon', + # 'lfs': 'coldfront.plugins.lustre', + } + rtype = next((k for k in resources_plugins), None) + if not rtype: + err = ('You cannot auto-update non-isilon resources at this ' + 'time. Please manually update the resource before ' + 'approving this change request.') + messages.error(request, err) + return self.redirect_to_detail(pk) + # get new quota value + new_quota = next(( + a for a in attrs_to_change if a['name'] == 'Storage Quota (TB)'), None) + if not new_quota: + err = ('You can only auto-update resource quotas at this ' + 'time. Please manually update the resource before ' + 'approving this change request.') + messages.error(request, err) + return self.redirect_to_detail(pk) + + new_quota_value = new_quota['value'] + plugin = resources_plugins[rtype] + if plugin in settings.INSTALLED_APPS: + # try to run the thing + try: + update_isilon_allocation_quota( + alloc_change_obj.allocation, new_quota_value + ) + except Exception as e: + err = ("An error was encountered while auto-updating" + "the allocation quota. Please contact Coldfront " + "administration and/or manually update the allocation.") + messages.error(request, err) + return self.redirect_to_detail(pk) + else: + err = ("There is an issue with the configuration of" + "Coldfront's auto-updating capabilities. Please contact Coldfront " + "administration and/or manually update the allocation.") + messages.error(request, err) + return self.redirect_to_detail(pk) + for attribute_change in attr_changes: new_value = attribute_change.new_value attribute_change.allocation_attribute.value = new_value diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 5f1438e42..76bbbda4c 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -73,6 +73,7 @@ def update_isilon_allocation_quota(allocation, quota): except ApiException as e: err = f"ERROR: could not update quota for {allocation} to {quota} - {e}" print_log_error(e, err) + raise def print_log_error(e, message): print(f'ERROR: {message} - {e}') @@ -88,4 +89,3 @@ def update_coldfront_quota_and_usage(alloc, usage_attribute_type, value_list): usage.value = value_list[1] usage.save() return usage_attribute - From 38e8de4a813978dd753f3db76d61b4eeac4d2b32 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 16:57:31 -0800 Subject: [PATCH 11/28] minor formatting --- coldfront/core/allocation/forms.py | 38 +++++++++++++++--------------- coldfront/core/allocation/views.py | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index 8de865baf..bba826ba4 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -81,7 +81,7 @@ class AllocationForm(forms.Form): ) expense_code = ExpenseCodeField( - label="...or add a new 33 digit expense code manually here.", required=False + label='...or add a new 33 digit expense code manually here.', required=False ) tier = forms.ModelChoiceField( @@ -110,7 +110,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): project_obj = get_object_or_404(Project, pk=project_pk) self.fields['tier'].queryset = get_user_resources(request_user).filter( resource_type__name='Storage Tier' - ).order_by(Lower("name")) + ).order_by(Lower('name')) existing_expense_codes = [(None, '------')] + [ (a.code, f'{a.code} ({a.name})') for a in Account.objects.filter( userproductaccount__is_valid=1, @@ -120,7 +120,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): self.fields['existing_expense_codes'].choices = existing_expense_codes user_query_set = project_obj.projectuser_set.select_related('user').filter( status__name__in=['Active', ] - ).order_by("user__username").exclude(user=project_obj.pi) + ).order_by('user__username').exclude(user=project_obj.pi) # if user_query_set: # self.fields['users'].choices = ((user.user.username, "%s %s (%s)" % ( # u.user.first_name, u.user.last_name, u.user.username)) for u in user_query_set) @@ -131,8 +131,8 @@ def __init__(self, request_user, project_pk, *args, **kwargs): def clean(self): cleaned_data = super().clean() # Remove all dashes from the input string to count the number of digits - expense_code = cleaned_data.get("expense_code") - existing_expense_codes = cleaned_data.get("existing_expense_codes") + expense_code = cleaned_data.get('expense_code') + existing_expense_codes = cleaned_data.get('existing_expense_codes') trues = sum(x for x in [ (expense_code not in ['', '------']), (existing_expense_codes not in ['', '------']), @@ -140,8 +140,8 @@ def clean(self): digits_only = lambda v: re.sub(r'[^0-9xX]', '', v) if trues != 1: self.add_error( - "existing_expense_codes", - "You must either select an existing expense code or manually enter a new one." + 'existing_expense_codes', + 'You must either select an existing expense code or manually enter a new one.' ) elif expense_code and expense_code != '------': @@ -169,7 +169,7 @@ class AllocationUpdateForm(forms.Form): label='Resource', queryset=Resource.objects.all(), required=False ) status = forms.ModelChoiceField( - queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")), empty_label=None) + queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')), empty_label=None) start_date = forms.DateField( label='Start Date', widget=forms.DateInput(attrs={'class': 'datepicker'}), @@ -204,8 +204,8 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - start_date = cleaned_data.get("start_date") - end_date = cleaned_data.get("end_date") + start_date = cleaned_data.get('start_date') + end_date = cleaned_data.get('end_date') if start_date and end_date and end_date < start_date: raise forms.ValidationError('End date cannot be less than start date') return cleaned_data @@ -214,7 +214,7 @@ def clean(self): class AllocationInvoiceUpdateForm(forms.Form): status = forms.ModelChoiceField(queryset=AllocationStatusChoice.objects.filter( name__in=['Payment Pending', 'Payment Requested', 'Payment Declined', 'Paid'] - ).order_by(Lower("name")), empty_label=None) + ).order_by(Lower('name')), empty_label=None) class AllocationAddUserForm(forms.Form): @@ -250,16 +250,16 @@ class AllocationSearchForm(forms.Form): username = forms.CharField(label='Username', max_length=100, required=False) resource_type = forms.ModelChoiceField( label='Resource Type', - queryset=ResourceType.objects.all().order_by(Lower("name")), + queryset=ResourceType.objects.all().order_by(Lower('name')), required=False) resource_name = forms.ModelMultipleChoiceField( label='Resource Name', queryset=Resource.objects.filter( - is_allocatable=True).order_by(Lower("name")), + is_allocatable=True).order_by(Lower('name')), required=False) allocation_attribute_name = forms.ModelChoiceField( label='Allocation Attribute Name', - queryset=AllocationAttributeType.objects.all().order_by(Lower("name")), + queryset=AllocationAttributeType.objects.all().order_by(Lower('name')), required=False) allocation_attribute_value = forms.CharField( label='Allocation Attribute Value', max_length=100, required=False) @@ -273,7 +273,7 @@ class AllocationSearchForm(forms.Form): required=False) status = forms.ModelMultipleChoiceField( widget=forms.CheckboxSelectMultiple, - queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")), + queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')), required=False) show_all_allocations = forms.BooleanField(initial=False, required=False) @@ -323,7 +323,7 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - if cleaned_data.get('new_value') != "": + if cleaned_data.get('new_value') != '': allocation_attribute = AllocationAttribute.objects.get(pk=cleaned_data.get('pk')) allocation_attribute.value = cleaned_data.get('new_value') allocation_attribute.clean() @@ -351,10 +351,10 @@ def clean(self): class AllocationChangeForm(forms.Form): EXTENSION_CHOICES = [ - (0, "No Extension") + (0, 'No Extension') ] for choice in ALLOCATION_CHANGE_REQUEST_EXTENSION_DAYS: - EXTENSION_CHOICES.append((choice, f"{choice} days")) + EXTENSION_CHOICES.append((choice, f'{choice} days')) end_date_extension = forms.TypedChoiceField( label='Request End Date Extension', @@ -378,7 +378,7 @@ class AllocationChangeNoteForm(forms.Form): label='Notes', required=False, widget=forms.Textarea, - help_text="Leave any feedback about the allocation change request.") + help_text='Leave any feedback about the allocation change request.') ALLOCATION_AUTOUPDATE_OPTIONS = [ diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index f63d54123..27c9f4d33 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -1896,7 +1896,7 @@ def post(self, request, *args, **kwargs): messages.error(request, err) return self.redirect_to_detail(pk) else: - err = ("There is an issue with the configuration of" + err = ("There is an issue with the configuration of " "Coldfront's auto-updating capabilities. Please contact Coldfront " "administration and/or manually update the allocation.") messages.error(request, err) From 4f149795a3e5cfa1c3397aa260128900af3611a4 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 18 Jan 2024 20:54:19 -0500 Subject: [PATCH 12/28] add isilon plugin --- coldfront/config/plugins/isilon.py | 8 ++++++++ coldfront/config/settings.py | 19 ++++++++++--------- coldfront/plugins/isilon/__init__.py | 0 coldfront/plugins/isilon/utils.py | 15 +++++++++++++++ requirements.txt | 1 + 5 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 coldfront/config/plugins/isilon.py create mode 100644 coldfront/plugins/isilon/__init__.py create mode 100644 coldfront/plugins/isilon/utils.py diff --git a/coldfront/config/plugins/isilon.py b/coldfront/config/plugins/isilon.py new file mode 100644 index 000000000..91c85f00a --- /dev/null +++ b/coldfront/config/plugins/isilon.py @@ -0,0 +1,8 @@ +from coldfront.config.env import ENV +from coldfront.config.logging import LOGGING +from coldfront.config.base import INSTALLED_APPS + +INSTALLED_APPS += [ 'coldfront.plugins.isilon' ] +ISILON_USER = ENV.str('ISILON_USER') +ISILON_PASS = ENV.str('ISILON_PASS') + diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index f6d980eb9..05ceeaaef 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -18,20 +18,21 @@ # ColdFront plugin settings plugin_configs = { - 'PLUGIN_SLURM': 'plugins/slurm.py', - 'PLUGIN_IQUOTA': 'plugins/iquota.py', - 'PLUGIN_FREEIPA': 'plugins/freeipa.py', - 'PLUGIN_SYSMON': 'plugins/system_montior.py', - 'PLUGIN_XDMOD': 'plugins/xdmod.py', + 'PLUGIN_API': 'plugins/api.py', 'PLUGIN_AUTH_OIDC': 'plugins/openid.py', 'PLUGIN_AUTH_LDAP': 'plugins/ldap.py', - 'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py', - 'PLUGIN_API': 'plugins/api.py', 'PLUGIN_LDAP': 'plugins/ldap_fasrc.py', - 'PLUGIN_SFTOCF': 'plugins/sftocf.py', + 'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py', 'PLUGIN_FASRC': 'plugins/fasrc.py', - 'PLUGIN_IFX': 'plugins/ifx.py', 'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py', + 'PLUGIN_FREEIPA': 'plugins/freeipa.py', + 'PLUGIN_IFX': 'plugins/ifx.py', + 'PLUGIN_ISILON': 'plugins/isilon.py', + 'PLUGIN_IQUOTA': 'plugins/iquota.py', + 'PLUGIN_SFTOCF': 'plugins/sftocf.py', + 'PLUGIN_SYSMON': 'plugins/system_montior.py', + 'PLUGIN_SLURM': 'plugins/slurm.py', + 'PLUGIN_XDMOD': 'plugins/xdmod.py', } # This allows plugins to be enabled via environment variables. Can alternatively diff --git a/coldfront/plugins/isilon/__init__.py b/coldfront/plugins/isilon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py new file mode 100644 index 000000000..8009c1c7f --- /dev/null +++ b/coldfront/plugins/isilon/utils.py @@ -0,0 +1,15 @@ +import isilon_sdk.v9_5_0 +from isilon_sdk.v9_5_0.rest import ApiException + +from coldfront.config.env import ENV +from coldfront.core.utils.common import import_from_settings + +def connect(): + configuration = isilon_sdk.v9_5_0.Configuration() + configuration.host = 'http://holy-isilon01.rc.fas.harvard.edu:8080' + configuration.username = import_from_settings('ISILON_USER') + configuration.password = import_from_settings('ISILON_PASS') + configuration.verify_ssl = False + api_client = isilon_sdk.v9_5_0.ApiClient(configuration) + api_instance = isilon_sdk.v9_5_0.ProtocolsApi(api_client) + return api_instance diff --git a/requirements.txt b/requirements.txt index b5c084c34..55fa5334a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ factory-boy==3.2.1 future==0.18.3 humanize==4.6.0 idna==3.4 +isilon-sdk pyparsing==3.0.9 python-dateutil==2.8.2 python-memcached==1.59 From a8253c1847ab292f7bd992d2152a73dcfb9da367 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 23 Jan 2024 14:48:26 -0800 Subject: [PATCH 13/28] add basic quota update function --- coldfront/plugins/isilon/utils.py | 84 ++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 8009c1c7f..a36327574 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,15 +1,83 @@ -import isilon_sdk.v9_5_0 -from isilon_sdk.v9_5_0.rest import ApiException +import isilon_sdk.v9_3_0 +from isilon_sdk.v9_3_0.rest import ApiException +import requests from coldfront.config.env import ENV +from coldfront.core.allocation.models import Allocation, AllocationAttributeType +from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings -def connect(): - configuration = isilon_sdk.v9_5_0.Configuration() - configuration.host = 'http://holy-isilon01.rc.fas.harvard.edu:8080' + +def connect(cluster_name): + configuration = isilon_sdk.v9_3_0.Configuration() + configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' configuration.username = import_from_settings('ISILON_USER') configuration.password = import_from_settings('ISILON_PASS') configuration.verify_ssl = False - api_client = isilon_sdk.v9_5_0.ApiClient(configuration) - api_instance = isilon_sdk.v9_5_0.ProtocolsApi(api_client) - return api_instance + api_client = isilon_sdk.v9_3_0.ApiClient(configuration) + return api_client + + +def update_quota_and_usage(alloc, usage_attribute_type, value_list): + usage_attribute, _ = alloc.allocation.allocationattribute_set.get_or_create( + allocation_attribute_type=usage_attribute_type + ) + usage_attribute.value = value_list[0] + usage_attribute.save() + usage = usage_attribute.allocationattributeusage + usage.value = value_list[1] + usage.save() + return usage_attribute + +def update_quotas_usages(): + """For all active tier1 allocations, update quota and usage + 1. run a query that collects all active tier1 allocations + """ + quota_bytes_attributetype = AllocationAttributeType.objects.get( + name='Quota_In_Bytes') + quota_tbs_attributetype = AllocationAttributeType.objects.get( + name='Storage Quota (TB)') + # create isilon connections to all isilon clusters in coldfront + isilon_resources = Resource.objects.filter(name__contains='tier1') + isilon_clusters = {} + for resource in isilon_resources: + resource_name = resource.name.split('/')[0] + # try connecting to the cluster. If it fails, display an error and + # replace the resource with a dummy resource + try: + api_client = connect(resource_name) + isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) + except Exception as e: + print(f'Could not connect to {resource_name} - will not update quotas for allocations on this resource') + print(e) + # isilon_clusters[resource.name] = None + + isilon_allocations = Allocation.objects.filter( + is_active=True, + resource__in=isilon_clusters.keys(), + ) + + for allocation in isilon_allocations: + # get the api instance for this allocation. If it doesn't exist, skip + api_instance = isilon_clusters[allocation.resource.name] + if not api_instance: + continue + try: + api_response = api_instance.get_quota_quota( + path=allocation.resource.path, + recurse_path_children=True, + ) + except ApiException as e: + print("Exception when calling QuotaApi->list_quotas: %s\n" % e) + continue + # update the quota and usage for this allocation + quota = api_response['thresholds']['hard'] + usage = api_response['usage']['fslogical'] + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) From 3bc8f1297c10893b9b0e96a333de2b9d42f7d9d7 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Fri, 26 Jan 2024 12:51:58 -0800 Subject: [PATCH 14/28] add command --- .../management/commands/pull_isilon_quotas.py | 12 ++++++++++++ coldfront/plugins/isilon/utils.py | 14 +++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py diff --git a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py new file mode 100644 index 000000000..0d9f42961 --- /dev/null +++ b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from coldfront.plugins.isilon.utils import update_quotas_usages + +class Command(BaseCommand): + """ + Pull Isilon quotas + """ + help = 'Pull Isilon quotas' + + def handle(self, *args, **kwargs): + update_quotas_usages() diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index a36327574..71004b0dd 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,12 +1,13 @@ +import logging + import isilon_sdk.v9_3_0 from isilon_sdk.v9_3_0.rest import ApiException -import requests -from coldfront.config.env import ENV from coldfront.core.allocation.models import Allocation, AllocationAttributeType from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings +logger = logging.getLogger(__name__) def connect(cluster_name): configuration = isilon_sdk.v9_3_0.Configuration() @@ -48,8 +49,9 @@ def update_quotas_usages(): api_client = connect(resource_name) isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) except Exception as e: - print(f'Could not connect to {resource_name} - will not update quotas for allocations on this resource') - print(e) + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + logger.warning("%s Error: %s", message, e) + print(f"{message} Error: {e}") # isilon_clusters[resource.name] = None isilon_allocations = Allocation.objects.filter( @@ -68,7 +70,9 @@ def update_quotas_usages(): recurse_path_children=True, ) except ApiException as e: - print("Exception when calling QuotaApi->list_quotas: %s\n" % e) + message = f'Exception when calling QuotaApi->list_quotas: {e}' + print(message) + logger.warning(message) continue # update the quota and usage for this allocation quota = api_response['thresholds']['hard'] From 3df629b4b0cf9cda9a0b41d5b53fdcbb1baea831 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 09:25:57 -0800 Subject: [PATCH 15/28] change structure of allocation collection --- coldfront/plugins/isilon/utils.py | 57 ++++++++++++++----------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 71004b0dd..48fae3323 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -40,48 +40,43 @@ def update_quotas_usages(): name='Storage Quota (TB)') # create isilon connections to all isilon clusters in coldfront isilon_resources = Resource.objects.filter(name__contains='tier1') - isilon_clusters = {} for resource in isilon_resources: resource_name = resource.name.split('/')[0] # try connecting to the cluster. If it fails, display an error and # replace the resource with a dummy resource try: api_client = connect(resource_name) - isilon_clusters[resource.name] = isilon_sdk.v9_3_0.QuotaApi(api_client) + api_instance = isilon_sdk.v9_3_0.QuotaApi(api_client) except Exception as e: message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' logger.warning("%s Error: %s", message, e) print(f"{message} Error: {e}") # isilon_clusters[resource.name] = None - - isilon_allocations = Allocation.objects.filter( - is_active=True, - resource__in=isilon_clusters.keys(), - ) - - for allocation in isilon_allocations: - # get the api instance for this allocation. If it doesn't exist, skip - api_instance = isilon_clusters[allocation.resource.name] - if not api_instance: - continue - try: - api_response = api_instance.get_quota_quota( - path=allocation.resource.path, - recurse_path_children=True, - ) - except ApiException as e: - message = f'Exception when calling QuotaApi->list_quotas: {e}' - print(message) - logger.warning(message) continue - # update the quota and usage for this allocation - quota = api_response['thresholds']['hard'] - usage = api_response['usage']['fslogical'] - quota_tb = quota / 1024 / 1024 / 1024 / 1024 - usage_tb = usage / 1024 / 1024 / 1024 / 1024 - update_quota_and_usage( - allocation, quota_bytes_attributetype, [quota, usage] + + # get all active allocations for this resource + isilon_allocations = Allocation.objects.filter( + status__name='Active', + resources__name=resource.name, ) - update_quota_and_usage( - allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + + # get all allocation quotas and usoges + api_response = api_instance.list_quota_quotas( + path='/ifs/rc_labs/', + recurse_path_children=True, ) + + for allocation in isilon_allocations: + # get the api_response entry for this allocation. If it doesn't exist, skip + api_entry = next(e for e in api_response.quota_quotas if e['path'] == f'/ifs/{allocation.path}') + # update the quota and usage for this allocation + quota = api_entry['thresholds']['hard'] + usage = api_entry['usage']['fslogical'] + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) From 2dfbc8b2e3955d83c7ea6296c006a51c7babd9d9 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 16:20:14 -0500 Subject: [PATCH 16/28] make pipeline operational --- coldfront/plugins/isilon/utils.py | 46 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 48fae3323..9cca97cfb 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -20,7 +20,7 @@ def connect(cluster_name): def update_quota_and_usage(alloc, usage_attribute_type, value_list): - usage_attribute, _ = alloc.allocation.allocationattribute_set.get_or_create( + usage_attribute, _ = alloc.allocationattribute_set.get_or_create( allocation_attribute_type=usage_attribute_type ) usage_attribute.value = value_list[0] @@ -41,6 +41,7 @@ def update_quotas_usages(): # create isilon connections to all isilon clusters in coldfront isilon_resources = Resource.objects.filter(name__contains='tier1') for resource in isilon_resources: + report = {"complete": 0, "no entry": [], "empty quota": []} resource_name = resource.name.split('/')[0] # try connecting to the cluster. If it fails, display an error and # replace the resource with a dummy resource @@ -61,17 +62,40 @@ def update_quotas_usages(): ) # get all allocation quotas and usoges - api_response = api_instance.list_quota_quotas( - path='/ifs/rc_labs/', - recurse_path_children=True, - ) - + try: + rc_labs = api_instance.list_quota_quotas( + path='/ifs/rc_labs/', + recurse_path_children=True, + ) + l3_labs = api_instance.list_quota_quotas( + path='/ifs/rc_fasse_labs/', + recurse_path_children=True, + ) + except Exception as e: + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + logger.warning("%s Error: %s", message, e) + print(f"{message} Error: {e}") + # isilon_clusters[resource.name] = None + continue + quotas = rc_labs.quotas + l3_labs.quotas for allocation in isilon_allocations: # get the api_response entry for this allocation. If it doesn't exist, skip - api_entry = next(e for e in api_response.quota_quotas if e['path'] == f'/ifs/{allocation.path}') + + try: + api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') + except StopIteration as e: + logger.error('no isilon quota entry for allocation %s', allocation) + print('no isilon quota entry for allocation', allocation) + report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue # update the quota and usage for this allocation - quota = api_entry['thresholds']['hard'] - usage = api_entry['usage']['fslogical'] + quota = api_entry.thresholds.hard + usage = api_entry.usage.fslogical + if quota is None: + logger.error('no hard threshold set for allocation %s', allocation) + print('no hard threshold set for allocation', allocation) + report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue quota_tb = quota / 1024 / 1024 / 1024 / 1024 usage_tb = usage / 1024 / 1024 / 1024 / 1024 update_quota_and_usage( @@ -80,3 +104,7 @@ def update_quotas_usages(): update_quota_and_usage( allocation, quota_tbs_attributetype, [quota_tb, usage_tb] ) + print("SUCCESS:update for allocation", allocation, "complete") + report['complete'] += 1 + print(report) + logger.warning("isilon update report for %s: %s", resource_name, report) From ff330141f759771bb57d576dd8973c25c963549a Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 30 Jan 2024 10:14:41 -0500 Subject: [PATCH 17/28] config fixes --- coldfront/config/plugins/isilon.py | 4 ++-- coldfront/config/settings.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/config/plugins/isilon.py b/coldfront/config/plugins/isilon.py index 91c85f00a..6b1169bc0 100644 --- a/coldfront/config/plugins/isilon.py +++ b/coldfront/config/plugins/isilon.py @@ -3,6 +3,6 @@ from coldfront.config.base import INSTALLED_APPS INSTALLED_APPS += [ 'coldfront.plugins.isilon' ] -ISILON_USER = ENV.str('ISILON_USER') -ISILON_PASS = ENV.str('ISILON_PASS') +ISILON_USER = ENV.str('ISILON_USER', '') +ISILON_PASS = ENV.str('ISILON_PASS', '') diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 05ceeaaef..725217259 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -30,7 +30,7 @@ 'PLUGIN_ISILON': 'plugins/isilon.py', 'PLUGIN_IQUOTA': 'plugins/iquota.py', 'PLUGIN_SFTOCF': 'plugins/sftocf.py', - 'PLUGIN_SYSMON': 'plugins/system_montior.py', + 'PLUGIN_SYSMON': 'plugins/system_monitor.py', 'PLUGIN_SLURM': 'plugins/slurm.py', 'PLUGIN_XDMOD': 'plugins/xdmod.py', } From bf4b5311349793edad1eb3dd0a9b61a49e0654af Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 30 Jan 2024 10:27:05 -0500 Subject: [PATCH 18/28] allocation admin changes --- coldfront/core/allocation/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/allocation/admin.py b/coldfront/core/allocation/admin.py index da2509250..ace54c28e 100644 --- a/coldfront/core/allocation/admin.py +++ b/coldfront/core/allocation/admin.py @@ -29,7 +29,7 @@ class AllocationStatusChoiceAdmin(admin.ModelAdmin): class AllocationUserInline(admin.TabularInline): model = AllocationUser extra = 0 - fields = ('id', 'user', 'status', 'usage',) + fields = ('id', 'user', 'status', 'usage', 'usage_bytes', 'unit') class AllocationAttributeInline(admin.TabularInline): From 82bb7c37591da78e3c7ed754ddb98fe2819ecdd1 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Sat, 27 Jan 2024 13:24:02 -0800 Subject: [PATCH 19/28] add djangoq task --- coldfront/plugins/isilon/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 coldfront/plugins/isilon/tasks.py diff --git a/coldfront/plugins/isilon/tasks.py b/coldfront/plugins/isilon/tasks.py new file mode 100644 index 000000000..8796ae153 --- /dev/null +++ b/coldfront/plugins/isilon/tasks.py @@ -0,0 +1,6 @@ +from coldfront.plugins.isilon.utils import update_quotas_usages + +def pull_isilon_quotas(): + """Pull Isilon quotas + """ + update_quotas_usages() From 4654ae1cb76b6f369dff7f59702574860e7bcc08 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 12:17:00 -0800 Subject: [PATCH 20/28] restructure isilon utils --- .../management/commands/pull_isilon_quotas.py | 84 ++++++++- coldfront/plugins/isilon/utils.py | 161 ++++++++---------- 2 files changed, 153 insertions(+), 92 deletions(-) diff --git a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py index 0d9f42961..33e052df0 100644 --- a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py +++ b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py @@ -1,6 +1,16 @@ +import logging + from django.core.management.base import BaseCommand -from coldfront.plugins.isilon.utils import update_quotas_usages +from coldfront.core.allocation.models import Allocation, AllocationAttributeType +from coldfront.core.resource.models import Resource +from coldfront.plugins.isilon.utils import ( + IsilonConnection, + print_log_error, + update_coldfront_quota_and_usage, +) + +logger = logging.getLogger(__name__) class Command(BaseCommand): """ @@ -9,4 +19,74 @@ class Command(BaseCommand): help = 'Pull Isilon quotas' def handle(self, *args, **kwargs): - update_quotas_usages() + """For all active tier1 allocations, update quota and usage + 1. run a query that collects all active tier1 allocations + """ + quota_bytes_attributetype = AllocationAttributeType.objects.get( + name='Quota_In_Bytes') + quota_tbs_attributetype = AllocationAttributeType.objects.get( + name='Storage Quota (TB)') + # create isilon connections to all isilon clusters in coldfront + isilon_resources = Resource.objects.filter(name__contains='tier1') + for resource in isilon_resources: + report = {"complete": 0, "no entry": [], "empty quota": []} + resource_name = resource.name.split('/')[0] + # try connecting to the cluster. If it fails, display an error and + # replace the resource with a dummy resource + try: + api_instance = IsilonConnection(resource_name) + except Exception as e: + message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + print_log_error(e, message) + # isilon_clusters[resource.name] = None + continue + + # get all active allocations for this resource + isilon_allocations = Allocation.objects.filter( + status__name='Active', + resources__name=resource.name, + ) + + # get all allocation quotas and usoges + try: + rc_labs = api_instance.quota_client.list_quota_quotas( + path='/ifs/rc_labs/', recurse_path_children=True, + ) + l3_labs = api_instance.quota_client.list_quota_quotas( + path='/ifs/rc_fasse_labs/', recurse_path_children=True, + ) + except Exception as e: + err = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' + print_log_error(e, err) + # isilon_clusters[resource.name] = None + continue + quotas = rc_labs.quotas + l3_labs.quotas + for allocation in isilon_allocations: + # get the api_response entry for this allocation. If it doesn't exist, skip + try: + api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') + except StopIteration as e: + err = f'no isilon quota entry for allocation {allocation}' + print_log_error(e, err) + report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue + # update the quota and usage for this allocation + quota = api_entry.thresholds.hard + usage = api_entry.usage.fslogical + if quota is None: + err = f'no hard threshold set for allocation {allocation}' + print_log_error(None, err) + report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') + continue + quota_tb = quota / 1024 / 1024 / 1024 / 1024 + usage_tb = usage / 1024 / 1024 / 1024 / 1024 + update_coldfront_quota_and_usage( + allocation, quota_bytes_attributetype, [quota, usage] + ) + update_coldfront_quota_and_usage( + allocation, quota_tbs_attributetype, [quota_tb, usage_tb] + ) + print("SUCCESS:update for allocation", allocation, "complete") + report['complete'] += 1 + print(report) + logger.warning("isilon update report for %s: %s", resource_name, report) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 9cca97cfb..5f1438e42 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -1,25 +1,84 @@ import logging -import isilon_sdk.v9_3_0 +import isilon_sdk.v9_3_0 as isilon_api from isilon_sdk.v9_3_0.rest import ApiException -from coldfront.core.allocation.models import Allocation, AllocationAttributeType -from coldfront.core.resource.models import Resource from coldfront.core.utils.common import import_from_settings logger = logging.getLogger(__name__) -def connect(cluster_name): - configuration = isilon_sdk.v9_3_0.Configuration() - configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' - configuration.username = import_from_settings('ISILON_USER') - configuration.password = import_from_settings('ISILON_PASS') - configuration.verify_ssl = False - api_client = isilon_sdk.v9_3_0.ApiClient(configuration) - return api_client +class IsilonConnection: + """Convenience class containing methods for collecting data from an isilon cluster + """ + def __init__(self, cluster_name): + self.cluster_name = cluster_name + self.api_client = self.connect(cluster_name) + self.quota_client = isilon_api.QuotaApi(self.api_client) + self.pools_client = isilon_api.StoragepoolApi(self.api_client) + + def connect(self, cluster_name): + configuration = isilon_api.Configuration() + configuration.host = f'http://{cluster_name}01.rc.fas.harvard.edu:8080' + configuration.username = import_from_settings('ISILON_USER') + configuration.password = import_from_settings('ISILON_PASS') + configuration.verify_ssl = False + api_client = isilon_api.ApiClient(configuration) + return api_client + + def get_isilon_volume_unallocated_space(self): + """get the total unallocated space on a volume + calculated as total usable space minus sum of all quotas on the volume + """ + try: + quotas = self.quota_client.list_quota_quotas(type='directory') + quota_sum = sum( + [q.thresholds.hard for q in quotas.quotas if q.thresholds.hard]) + pool_query = self.pools_client.get_storagepool_storagepools( + toplevels=True) + total_space = pool_query.usage.usable_bytes + return total_space - quota_sum + except ApiException as e: + err = f'ERROR: could not get quota for {self.cluster_name} - {e}' + print_log_error(e, err) + return None + +def update_isilon_allocation_quota(allocation, quota): + """Update the quota for an allocation on an isilon cluster + + Parameters + ---------- + api_instance : isilon_api.QuotaApi + allocation : coldfront.core.allocation.models.Allocation + quota : int + """ + # make isilon connection to the allocation's resource + isilon_resource = allocation.resources.first().split('/')[0] + isilon_conn = IsilonConnection(isilon_resource) + path = f'/ifs/{allocation.path}' + + # check if enough space exists on the volume + quota_bytes = quota * 1024**3 + unallocated_space = isilon_conn.get_isilon_volume_unallocated_space() + allowable_space = unallocated_space * 0.8 + if allowable_space < quota_bytes: + raise ValueError( + 'ERROR: not enough space on volume to set quota to %s for %s' + % (quota, allocation) + ) + try: + isilon_conn.quota_client.update_quota_quota(path=path, threshold_hard=quota) + print(f"SUCCESS: updated quota for {allocation} to {quota}") + logger.info("SUCCESS: updated quota for %s to %s", allocation, quota) + except ApiException as e: + err = f"ERROR: could not update quota for {allocation} to {quota} - {e}" + print_log_error(e, err) -def update_quota_and_usage(alloc, usage_attribute_type, value_list): +def print_log_error(e, message): + print(f'ERROR: {message} - {e}') + logger.error("%s - %s", message, e) + +def update_coldfront_quota_and_usage(alloc, usage_attribute_type, value_list): usage_attribute, _ = alloc.allocationattribute_set.get_or_create( allocation_attribute_type=usage_attribute_type ) @@ -30,81 +89,3 @@ def update_quota_and_usage(alloc, usage_attribute_type, value_list): usage.save() return usage_attribute -def update_quotas_usages(): - """For all active tier1 allocations, update quota and usage - 1. run a query that collects all active tier1 allocations - """ - quota_bytes_attributetype = AllocationAttributeType.objects.get( - name='Quota_In_Bytes') - quota_tbs_attributetype = AllocationAttributeType.objects.get( - name='Storage Quota (TB)') - # create isilon connections to all isilon clusters in coldfront - isilon_resources = Resource.objects.filter(name__contains='tier1') - for resource in isilon_resources: - report = {"complete": 0, "no entry": [], "empty quota": []} - resource_name = resource.name.split('/')[0] - # try connecting to the cluster. If it fails, display an error and - # replace the resource with a dummy resource - try: - api_client = connect(resource_name) - api_instance = isilon_sdk.v9_3_0.QuotaApi(api_client) - except Exception as e: - message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - logger.warning("%s Error: %s", message, e) - print(f"{message} Error: {e}") - # isilon_clusters[resource.name] = None - continue - - # get all active allocations for this resource - isilon_allocations = Allocation.objects.filter( - status__name='Active', - resources__name=resource.name, - ) - - # get all allocation quotas and usoges - try: - rc_labs = api_instance.list_quota_quotas( - path='/ifs/rc_labs/', - recurse_path_children=True, - ) - l3_labs = api_instance.list_quota_quotas( - path='/ifs/rc_fasse_labs/', - recurse_path_children=True, - ) - except Exception as e: - message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - logger.warning("%s Error: %s", message, e) - print(f"{message} Error: {e}") - # isilon_clusters[resource.name] = None - continue - quotas = rc_labs.quotas + l3_labs.quotas - for allocation in isilon_allocations: - # get the api_response entry for this allocation. If it doesn't exist, skip - - try: - api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') - except StopIteration as e: - logger.error('no isilon quota entry for allocation %s', allocation) - print('no isilon quota entry for allocation', allocation) - report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - # update the quota and usage for this allocation - quota = api_entry.thresholds.hard - usage = api_entry.usage.fslogical - if quota is None: - logger.error('no hard threshold set for allocation %s', allocation) - print('no hard threshold set for allocation', allocation) - report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - quota_tb = quota / 1024 / 1024 / 1024 / 1024 - usage_tb = usage / 1024 / 1024 / 1024 / 1024 - update_quota_and_usage( - allocation, quota_bytes_attributetype, [quota, usage] - ) - update_quota_and_usage( - allocation, quota_tbs_attributetype, [quota_tb, usage_tb] - ) - print("SUCCESS:update for allocation", allocation, "complete") - report['complete'] += 1 - print(report) - logger.warning("isilon update report for %s: %s", resource_name, report) From 3c4f345d10e32d1ec5536087e69d64e531589cae Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 16:37:56 -0800 Subject: [PATCH 21/28] attach isilon auto-update to GUI --- coldfront/core/allocation/forms.py | 25 +++++++- .../allocation/allocation_change_detail.html | 6 +- coldfront/core/allocation/views.py | 60 ++++++++++++++++++- coldfront/plugins/isilon/utils.py | 2 +- 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index 592785795..8de865baf 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -40,10 +40,10 @@ def validate(self, value): "Input must consist only of digits (or x'es) and dashes." ) if len(digits_only(value)) != 33: - raise ValidationError("Input must contain exactly 33 digits.") + raise ValidationError('Input must contain exactly 33 digits.') if 'x' in digits_only(value)[:8]+digits_only(value)[12:]: raise ValidationError( - "xes are only allowed in place of the product code (the third grouping of characters in the code)" + 'xes are only allowed in place of the product code (the third grouping of characters in the code)' ) def clean(self, value): @@ -123,7 +123,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): ).order_by("user__username").exclude(user=project_obj.pi) # if user_query_set: # self.fields['users'].choices = ((user.user.username, "%s %s (%s)" % ( - # user.user.first_name, user.user.last_name, user.user.username)) for user in user_query_set) + # u.user.first_name, u.user.last_name, u.user.username)) for u in user_query_set) # self.fields['users'].help_text = '
Select users in your project to add to this allocation.' # else: # self.fields['users'].widget = forms.HiddenInput() @@ -380,6 +380,25 @@ class AllocationChangeNoteForm(forms.Form): widget=forms.Textarea, help_text="Leave any feedback about the allocation change request.") + +ALLOCATION_AUTOUPDATE_OPTIONS = [ + ('1', 'I have already manually modified the allocation.'), + ('2', 'I would like Coldfront to modify the allocation for me. If Coldfront experiences any issues with the modification process, I understand that I will need to modify the allocation manually instead.'), +] + +class AllocationAutoUpdateForm(forms.Form): + sheetcheck = forms.BooleanField( + label='I have ensured that enough space is available on this resource.' + ) + auto_update_opts = forms.ChoiceField( + label='How will this allocation be modified?', + required=True, + widget=forms.RadioSelect, + choices=ALLOCATION_AUTOUPDATE_OPTIONS, + ) + + + class AllocationAttributeCreateForm(forms.ModelForm): class Meta: model = AllocationAttribute diff --git a/coldfront/core/allocation/templates/allocation/allocation_change_detail.html b/coldfront/core/allocation/templates/allocation/allocation_change_detail.html index 6d8fcd82d..0bf4e15ca 100644 --- a/coldfront/core/allocation/templates/allocation/allocation_change_detail.html +++ b/coldfront/core/allocation/templates/allocation/allocation_change_detail.html @@ -193,8 +193,9 @@

Actio
- - {% csrf_token %} + {% csrf_token %} + {{autoupdate_form.sheetcheck | as_crispy_field}} + {{autoupdate_form.auto_update_opts | as_crispy_field}} {{note_form.notes | as_crispy_field}}
{% if allocation_change.status.name == 'Pending' %} @@ -205,7 +206,6 @@

Actio Update

-
{% endif %} diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index c3ca0948c..c3842159a 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -35,6 +35,7 @@ AllocationAttributeChangeForm, AllocationAttributeUpdateForm, AllocationForm, + AllocationAutoUpdateForm, AllocationInvoiceNoteDeleteForm, AllocationInvoiceUpdateForm, AllocationRemoveUserForm, @@ -72,6 +73,8 @@ from ifxbilling.models import Account, UserProductAccount if 'django_q' in settings.INSTALLED_APPS: from django_q.tasks import Task +if 'coldfront.plugins.isilon' in settings.INSTALLED_APPS: + from coldfront.plugins.isilon import update_isilon_allocation_quota ALLOCATION_ENABLE_ALLOCATION_RENEWAL = import_from_settings( 'ALLOCATION_ENABLE_ALLOCATION_RENEWAL', True) @@ -1718,9 +1721,12 @@ def get(self, request, *args, **kwargs): initial={'notes': allocation_change_obj.notes} ) + autoupdate_form = AllocationAutoUpdateForm() + context = self.get_context_data() context['allocation_change_form'] = allocation_change_form + context['autoupdate_form'] = autoupdate_form context['note_form'] = note_form return render(request, self.template_name, context) @@ -1765,7 +1771,10 @@ def post(self, request, *args, **kwargs): context['allocation_change_form'] = allocation_change_form return render(request, self.template_name, context) + action = request.POST.get('action') + + autoupdate_form = AllocationAutoUpdateForm(request.POST) if action not in ['update', 'approve', 'deny']: return HttpResponseBadRequest('Invalid request') @@ -1845,9 +1854,54 @@ def post(self, request, *args, **kwargs): alloc_change_obj.allocation.save() if attrs_to_change: - attr_changes = ( - alloc_change_obj.allocationattributechangerequest_set.all() - ) + attr_changes = alloc_change_obj.allocationattributechangerequest_set.all() + + autoupdate_choice = autoupdate_form.data.get('auto_update_opts') + if autoupdate_choice == '2': + # check resource type, see if appropriate plugin is available + resource = alloc_change_obj.allocation.resources.first().name + resources_plugins = { + 'isilon': 'coldfront.plugins.isilon', + # 'lfs': 'coldfront.plugins.lustre', + } + rtype = next((k for k in resources_plugins), None) + if not rtype: + err = ('You cannot auto-update non-isilon resources at this ' + 'time. Please manually update the resource before ' + 'approving this change request.') + messages.error(request, err) + return self.redirect_to_detail(pk) + # get new quota value + new_quota = next(( + a for a in attrs_to_change if a['name'] == 'Storage Quota (TB)'), None) + if not new_quota: + err = ('You can only auto-update resource quotas at this ' + 'time. Please manually update the resource before ' + 'approving this change request.') + messages.error(request, err) + return self.redirect_to_detail(pk) + + new_quota_value = new_quota['value'] + plugin = resources_plugins[rtype] + if plugin in settings.INSTALLED_APPS: + # try to run the thing + try: + update_isilon_allocation_quota( + alloc_change_obj.allocation, new_quota_value + ) + except Exception as e: + err = ("An error was encountered while auto-updating" + "the allocation quota. Please contact Coldfront " + "administration and/or manually update the allocation.") + messages.error(request, err) + return self.redirect_to_detail(pk) + else: + err = ("There is an issue with the configuration of" + "Coldfront's auto-updating capabilities. Please contact Coldfront " + "administration and/or manually update the allocation.") + messages.error(request, err) + return self.redirect_to_detail(pk) + for attribute_change in attr_changes: new_value = attribute_change.new_value attribute_change.allocation_attribute.value = new_value diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 5f1438e42..76bbbda4c 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -73,6 +73,7 @@ def update_isilon_allocation_quota(allocation, quota): except ApiException as e: err = f"ERROR: could not update quota for {allocation} to {quota} - {e}" print_log_error(e, err) + raise def print_log_error(e, message): print(f'ERROR: {message} - {e}') @@ -88,4 +89,3 @@ def update_coldfront_quota_and_usage(alloc, usage_attribute_type, value_list): usage.value = value_list[1] usage.save() return usage_attribute - From 5b08129c5eed6c79755abb33da4914ece7bcd99f Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 13 Feb 2024 16:57:31 -0800 Subject: [PATCH 22/28] minor formatting --- coldfront/core/allocation/forms.py | 38 +++++++++++++++--------------- coldfront/core/allocation/views.py | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index 8de865baf..bba826ba4 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -81,7 +81,7 @@ class AllocationForm(forms.Form): ) expense_code = ExpenseCodeField( - label="...or add a new 33 digit expense code manually here.", required=False + label='...or add a new 33 digit expense code manually here.', required=False ) tier = forms.ModelChoiceField( @@ -110,7 +110,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): project_obj = get_object_or_404(Project, pk=project_pk) self.fields['tier'].queryset = get_user_resources(request_user).filter( resource_type__name='Storage Tier' - ).order_by(Lower("name")) + ).order_by(Lower('name')) existing_expense_codes = [(None, '------')] + [ (a.code, f'{a.code} ({a.name})') for a in Account.objects.filter( userproductaccount__is_valid=1, @@ -120,7 +120,7 @@ def __init__(self, request_user, project_pk, *args, **kwargs): self.fields['existing_expense_codes'].choices = existing_expense_codes user_query_set = project_obj.projectuser_set.select_related('user').filter( status__name__in=['Active', ] - ).order_by("user__username").exclude(user=project_obj.pi) + ).order_by('user__username').exclude(user=project_obj.pi) # if user_query_set: # self.fields['users'].choices = ((user.user.username, "%s %s (%s)" % ( # u.user.first_name, u.user.last_name, u.user.username)) for u in user_query_set) @@ -131,8 +131,8 @@ def __init__(self, request_user, project_pk, *args, **kwargs): def clean(self): cleaned_data = super().clean() # Remove all dashes from the input string to count the number of digits - expense_code = cleaned_data.get("expense_code") - existing_expense_codes = cleaned_data.get("existing_expense_codes") + expense_code = cleaned_data.get('expense_code') + existing_expense_codes = cleaned_data.get('existing_expense_codes') trues = sum(x for x in [ (expense_code not in ['', '------']), (existing_expense_codes not in ['', '------']), @@ -140,8 +140,8 @@ def clean(self): digits_only = lambda v: re.sub(r'[^0-9xX]', '', v) if trues != 1: self.add_error( - "existing_expense_codes", - "You must either select an existing expense code or manually enter a new one." + 'existing_expense_codes', + 'You must either select an existing expense code or manually enter a new one.' ) elif expense_code and expense_code != '------': @@ -169,7 +169,7 @@ class AllocationUpdateForm(forms.Form): label='Resource', queryset=Resource.objects.all(), required=False ) status = forms.ModelChoiceField( - queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")), empty_label=None) + queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')), empty_label=None) start_date = forms.DateField( label='Start Date', widget=forms.DateInput(attrs={'class': 'datepicker'}), @@ -204,8 +204,8 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - start_date = cleaned_data.get("start_date") - end_date = cleaned_data.get("end_date") + start_date = cleaned_data.get('start_date') + end_date = cleaned_data.get('end_date') if start_date and end_date and end_date < start_date: raise forms.ValidationError('End date cannot be less than start date') return cleaned_data @@ -214,7 +214,7 @@ def clean(self): class AllocationInvoiceUpdateForm(forms.Form): status = forms.ModelChoiceField(queryset=AllocationStatusChoice.objects.filter( name__in=['Payment Pending', 'Payment Requested', 'Payment Declined', 'Paid'] - ).order_by(Lower("name")), empty_label=None) + ).order_by(Lower('name')), empty_label=None) class AllocationAddUserForm(forms.Form): @@ -250,16 +250,16 @@ class AllocationSearchForm(forms.Form): username = forms.CharField(label='Username', max_length=100, required=False) resource_type = forms.ModelChoiceField( label='Resource Type', - queryset=ResourceType.objects.all().order_by(Lower("name")), + queryset=ResourceType.objects.all().order_by(Lower('name')), required=False) resource_name = forms.ModelMultipleChoiceField( label='Resource Name', queryset=Resource.objects.filter( - is_allocatable=True).order_by(Lower("name")), + is_allocatable=True).order_by(Lower('name')), required=False) allocation_attribute_name = forms.ModelChoiceField( label='Allocation Attribute Name', - queryset=AllocationAttributeType.objects.all().order_by(Lower("name")), + queryset=AllocationAttributeType.objects.all().order_by(Lower('name')), required=False) allocation_attribute_value = forms.CharField( label='Allocation Attribute Value', max_length=100, required=False) @@ -273,7 +273,7 @@ class AllocationSearchForm(forms.Form): required=False) status = forms.ModelMultipleChoiceField( widget=forms.CheckboxSelectMultiple, - queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")), + queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')), required=False) show_all_allocations = forms.BooleanField(initial=False, required=False) @@ -323,7 +323,7 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - if cleaned_data.get('new_value') != "": + if cleaned_data.get('new_value') != '': allocation_attribute = AllocationAttribute.objects.get(pk=cleaned_data.get('pk')) allocation_attribute.value = cleaned_data.get('new_value') allocation_attribute.clean() @@ -351,10 +351,10 @@ def clean(self): class AllocationChangeForm(forms.Form): EXTENSION_CHOICES = [ - (0, "No Extension") + (0, 'No Extension') ] for choice in ALLOCATION_CHANGE_REQUEST_EXTENSION_DAYS: - EXTENSION_CHOICES.append((choice, f"{choice} days")) + EXTENSION_CHOICES.append((choice, f'{choice} days')) end_date_extension = forms.TypedChoiceField( label='Request End Date Extension', @@ -378,7 +378,7 @@ class AllocationChangeNoteForm(forms.Form): label='Notes', required=False, widget=forms.Textarea, - help_text="Leave any feedback about the allocation change request.") + help_text='Leave any feedback about the allocation change request.') ALLOCATION_AUTOUPDATE_OPTIONS = [ diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index c3842159a..0c881cecc 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -1896,7 +1896,7 @@ def post(self, request, *args, **kwargs): messages.error(request, err) return self.redirect_to_detail(pk) else: - err = ("There is an issue with the configuration of" + err = ("There is an issue with the configuration of " "Coldfront's auto-updating capabilities. Please contact Coldfront " "administration and/or manually update the allocation.") messages.error(request, err) From 1ad8e6394c76060e302a625e635bcd387e048c91 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 15 Feb 2024 12:15:44 -0800 Subject: [PATCH 23/28] update pull_resource_data --- coldfront/core/project/views.py | 6 +- .../management/commands/pull_resource_data.py | 84 +++++++++++++++++++ coldfront/plugins/fasrc/tasks.py | 7 +- coldfront/plugins/isilon/utils.py | 48 ++++++----- .../commands/id_add_new_projects.py | 2 +- coldfront/plugins/ldap/utils.py | 62 +++++++++----- coldfront/plugins/sftocf/tasks.py | 5 -- coldfront/plugins/sftocf/utils.py | 49 ----------- 8 files changed, 160 insertions(+), 103 deletions(-) create mode 100644 coldfront/plugins/fasrc/management/commands/pull_resource_data.py diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 71d221c46..23f1fb6b7 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -163,7 +163,7 @@ def get_context_data(self, **kwargs): ) ) except ValueError: - err = "Allocation attribute '{}' is not an int but has a usage".format( + err = "Project attribute '{}' is not an int but has a usage".format( attribute.allocation_attribute_type.name ) logger.error(err) @@ -178,9 +178,7 @@ def get_context_data(self, **kwargs): context['mailto'] = 'mailto:' + ','.join([u.user.email for u in project_users]) - allocations = Allocation.objects.prefetch_related('resources').filter( - Q(project=self.object) - ) + allocations = self.object.allocation_set.prefetch_related('resources') allocation_history_records = self.return_status_change_records(allocations) if not self.request.user.is_superuser and not self.request.user.has_perm( diff --git a/coldfront/plugins/fasrc/management/commands/pull_resource_data.py b/coldfront/plugins/fasrc/management/commands/pull_resource_data.py new file mode 100644 index 000000000..d5d45e7a9 --- /dev/null +++ b/coldfront/plugins/fasrc/management/commands/pull_resource_data.py @@ -0,0 +1,84 @@ +import logging +from django.core.management.base import BaseCommand + +from coldfront.core.resource.models import Resource, ResourceAttributeType +from coldfront.plugins.isilon.utils import IsilonConnection +from coldfront.plugins.sftocf.utils import ( + StarFishServer, StarFishRedash, STARFISH_SERVER +) + +logger = logging.getLogger(__name__) + +def update_resource_attr_types_from_dict(resource, res_attr_type_dict): + for attr_name, attr_val in res_attr_type_dict.items(): + if attr_val: + attr_type_obj = ResourceAttributeType.objects.get(name=attr_name) + resource.resourceattribute_set.update_or_create( + resource_attribute_type=attr_type_obj, + defaults={'value': attr_val} + ) + +class Command(BaseCommand): + """Pull data from starfish and save to ResourceAttribute objects + """ + + def add_arguments(self, parser): + parser.add_argument( + 'source', + default='rest_api', + help='Do not make any changes to Starfish, just print what changes would be slated', + ) + + def handle(self, *args, **options): + source = options['source'] + if source == 'rest_api': + sf = StarFishServer(STARFISH_SERVER) + volumes = sf.get_volume_attributes() + volumes = [ + { + 'name': vol['vol'], + 'attrs': { + 'capacity_tb': vol['total_capacity']/(1024**4), + 'free_tb': vol['free_space']/(1024**4), + 'file_count': vol['number_of_files'], + } + } + for vol in volumes + ] + + elif source == 'redash': + sf = StarFishRedash(STARFISH_SERVER) + volumes = sf.get_vol_stats() + volumes = [ + { + 'name': vol['volume_name'], + 'attrs': { + 'capacity_tb': vol['capacity_TB'], + 'free_tb': vol['free_TB'], + 'file_count': vol['regular_files'], + } + } + for vol in volumes + ] + else: + raise ValueError('source must be "rest_api" or "redash"') + + # collect user and lab counts, allocation sizes for each volume + resources = Resource.objects.filter(resource_type__name='Storage') + for resource in resources: + resource_name = resource.name.split('/')[0] + if 'isilon' in resource.name: + isilon_api = IsilonConnection(resource_name) + isilon_capacity_tb = isilon_api.to_tb(isilon_api.total_space) + isilon_free_tb = isilon_api.to_tb(isilon_api.free_space) + attr_pairs = { + 'capacity_tb': isilon_capacity_tb, + 'free_tb': isilon_free_tb, + } + update_resource_attr_types_from_dict(resource, attr_pairs) + else: + volume = next([vol for vol in volumes if vol['name'] == resource_name], None) + if not volume: + logger.error('resource not found in starfish: %s', resource) + continue + update_resource_attr_types_from_dict(resource, volume['attrs']) diff --git a/coldfront/plugins/fasrc/tasks.py b/coldfront/plugins/fasrc/tasks.py index 6a115880f..1c63b37ec 100644 --- a/coldfront/plugins/fasrc/tasks.py +++ b/coldfront/plugins/fasrc/tasks.py @@ -18,6 +18,9 @@ def import_quotas(volumes=None): pull_push_quota_data() def id_import_allocations(): - """ID and import new allocations using ATT and Starfish data - """ + """ID and import new allocations using ATT and Starfish data""" management.call_command('id_import_new_allocations') + + +def pull_resource_data(): + management.call_command('pull_resource_data') diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 76bbbda4c..434377815 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -15,6 +15,8 @@ def __init__(self, cluster_name): self.api_client = self.connect(cluster_name) self.quota_client = isilon_api.QuotaApi(self.api_client) self.pools_client = isilon_api.StoragepoolApi(self.api_client) + self._total_space = None + self._used_space = None def connect(self, cluster_name): configuration = isilon_api.Configuration() @@ -25,23 +27,31 @@ def connect(self, cluster_name): api_client = isilon_api.ApiClient(configuration) return api_client - def get_isilon_volume_unallocated_space(self): - """get the total unallocated space on a volume - calculated as total usable space minus sum of all quotas on the volume - """ - try: - quotas = self.quota_client.list_quota_quotas(type='directory') - quota_sum = sum( - [q.thresholds.hard for q in quotas.quotas if q.thresholds.hard]) + @property + def total_space(self): + """total usable disk space""" + if self._total_space is None: pool_query = self.pools_client.get_storagepool_storagepools( toplevels=True) - total_space = pool_query.usage.usable_bytes - return total_space - quota_sum - except ApiException as e: - err = f'ERROR: could not get quota for {self.cluster_name} - {e}' - print_log_error(e, err) - return None + self._total_space = pool_query.usage.usable_bytes + return self._total_space + + @property + def used_space(self): + """space claimed by allocations""" + if self._used_space is None: + quotas = self.quota_client.list_quota_quotas(type='directory') + self._used_space = sum( + [q.thresholds.hard for q in quotas.quotas if q.thresholds.hard]) + return self._used_space + + @property + def free_space(self): + """total unallocated space on a volume""" + return self.total_space - self.used_space + def to_tb(self, bytes_value): + return bytes_value / (1024**3) def update_isilon_allocation_quota(allocation, quota): """Update the quota for an allocation on an isilon cluster @@ -59,7 +69,7 @@ def update_isilon_allocation_quota(allocation, quota): # check if enough space exists on the volume quota_bytes = quota * 1024**3 - unallocated_space = isilon_conn.get_isilon_volume_unallocated_space() + unallocated_space = isilon_conn.free_space allowable_space = unallocated_space * 0.8 if allowable_space < quota_bytes: raise ValueError( @@ -68,16 +78,16 @@ def update_isilon_allocation_quota(allocation, quota): ) try: isilon_conn.quota_client.update_quota_quota(path=path, threshold_hard=quota) - print(f"SUCCESS: updated quota for {allocation} to {quota}") - logger.info("SUCCESS: updated quota for %s to %s", allocation, quota) + print(f'SUCCESS: updated quota for {allocation} to {quota}') + logger.info('SUCCESS: updated quota for %s to %s', allocation, quota) except ApiException as e: - err = f"ERROR: could not update quota for {allocation} to {quota} - {e}" + err = f'ERROR: could not update quota for {allocation} to {quota} - {e}' print_log_error(e, err) raise def print_log_error(e, message): print(f'ERROR: {message} - {e}') - logger.error("%s - %s", message, e) + logger.error('%s - %s', message, e) def update_coldfront_quota_and_usage(alloc, usage_attribute_type, value_list): usage_attribute, _ = alloc.allocationattribute_set.get_or_create( diff --git a/coldfront/plugins/ldap/management/commands/id_add_new_projects.py b/coldfront/plugins/ldap/management/commands/id_add_new_projects.py index 724e60dcd..cb07199db 100644 --- a/coldfront/plugins/ldap/management/commands/id_add_new_projects.py +++ b/coldfront/plugins/ldap/management/commands/id_add_new_projects.py @@ -34,7 +34,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **kwargs): - groups = groups = kwargs['groups'] + groups = kwargs['groups'] if groups: groups = groups.split(",") # compare projects in AD to projects in coldfront diff --git a/coldfront/plugins/ldap/utils.py b/coldfront/plugins/ldap/utils.py index 608201583..97c957597 100644 --- a/coldfront/plugins/ldap/utils.py +++ b/coldfront/plugins/ldap/utils.py @@ -255,18 +255,20 @@ def user_valid(user): # and (user['accountExpires'][0].year == 1601 or user['accountExpires'][0] > timezone.now()) class GroupUserCollection: - """ - Class to hold a group and its members. + """Class to hold a group and its members. """ def __init__(self, group_name, ad_users, pi, project=None): self.name = group_name self.members = ad_users self.pi = pi self.project = project + self._current_ad_users = None @property def current_ad_users(self): - return [u for u in self.members if user_valid(u)] + if not self._current_ad_users: + self._current_ad_users = [u for u in self.members if user_valid(u)] + return self._current_ad_users @property def pi_is_active(self): @@ -357,7 +359,6 @@ def flatten(l): return [item for sublist in l for item in sublist] - def collect_update_project_status_membership(): """ Update Project and ProjectUser entries for existing Coldfront Projects using @@ -375,12 +376,18 @@ def collect_update_project_status_membership(): proj_membs_mans = {p: ad_conn.return_group_members_manager(p.title) for p in active_projects} proj_membs_mans, _ = cleaned_membership_query(proj_membs_mans) - groupusercollections = [GroupUserCollection(k.title, v[0], v[1], project=k) for k, v in proj_membs_mans.items()] + groupusercollections = [ + GroupUserCollection(k.title, v[0], v[1], project=k) for k, v in proj_membs_mans.items() + ] active_pi_groups, inactive_pi_groups = remove_inactive_disabled_managers(groupusercollections) projects_to_deactivate = [g.project for g in inactive_pi_groups] - Project.objects.bulk_update([Project(id=p.pk, status=ProjectStatusChoice.objects.get(name='Inactive')) - for p in projects_to_deactivate], ['status']) + Project.objects.bulk_update( + [ + Project(id=p.pk, status=ProjectStatusChoice.objects.get(name='Inactive')) + for p in projects_to_deactivate + ], ['status'] + ) logger.debug('projects_to_deactivate %s', projects_to_deactivate) if projects_to_deactivate: pis_to_deactivate = ProjectUser.objects.filter( @@ -389,7 +396,8 @@ def collect_update_project_status_membership(): )) logger.debug('pis_to_deactivate %s', pis_to_deactivate) pis_to_deactivate.update(status=ProjectUserStatusChoice.objects.get(name='Removed')) - logger.info('deactivated projects and pis: %s', [(pi.project.title, pi.user.username) for pi in pis_to_deactivate]) + logger.info('deactivated projects and pis: %s', + [(pi.project.title, pi.user.username) for pi in pis_to_deactivate]) ### identify PIs with incorrect roles and change their status ### projectuser_role_manager = ProjectUserRoleChoice.objects.get(name='Manager') @@ -400,8 +408,8 @@ def collect_update_project_status_membership(): Q(user__username=group.pi['sAMAccountName']) & ~Q(role=projectuser_role_manager)) for group in active_pi_groups) - ) ) + ) if pis_mislabeled: logger.info('Project PIs with incorrect labeling: %s', @@ -434,7 +442,9 @@ def collect_update_project_status_membership(): present_projectusers = group.project.projectuser_set.filter( user__in=present_project_ifxusers ) - logger.debug('present_users - ADUsers who have ifxuser accounts:\n%s', ad_users_not_added) + logger.debug( + 'present_users - ADUsers who have ifxuser accounts:\n%s', ad_users_not_added + ) if present_projectusers: logger.warning('found reactivated ADUsers for project %s: %s', group.project.title, [user.user.username for user in present_projectusers]) @@ -447,14 +457,14 @@ def collect_update_project_status_membership(): id__in=[pu.user.pk for pu in present_projectusers] ) logger.debug("missing_projectusers - ifxusers in present_project_ifxusers who are not ") - ProjectUser.objects.bulk_create([ProjectUser( - project=group.project, - user=user, - role=projectuser_role_user, - status=projectuserstatus_active - ) - for user in missing_projectusers - ]) + ProjectUser.objects.bulk_create([ + ProjectUser( + project=group.project, + user=user, + role=projectuser_role_user, + status=projectuserstatus_active + ) for user in missing_projectusers + ]) ### identify inactive ProjectUsers, slate for status change ### remove_projusers = group.project.projectuser_set.filter( @@ -502,8 +512,7 @@ def import_projects_projectusers(projects_list): def add_new_projects(groupusercollections, errortracker): - """create new Coldfront Projects and ProjectUsers from PI and AD user data - already collected from ATT. + """create new Projects and ProjectUsers from ATT-collected PI and AD user data """ # if PI is inactive, don't add project active_pi_groups, _ = remove_inactive_disabled_managers(groupusercollections) @@ -514,7 +523,10 @@ def add_new_projects(groupusercollections, errortracker): g for g in active_pi_groups if any(any(string in m for string in pi_groups) for m in g.pi['memberOf']) ] - logger.debug('active_invalid_pi_groups: %s', set(active_valid_pi_groups) - set(active_pi_groups)) + logger.debug( + 'active_invalid_pi_groups: %s', + set(active_valid_pi_groups) - set(active_pi_groups) + ) errortracker['pi_active_invalid'] = [group.name for group in active_pi_groups if group not in active_valid_pi_groups] @@ -543,7 +555,9 @@ def add_new_projects(groupusercollections, errortracker): for group in active_present_pi_groups: logger.debug('source: %s\n%s\n%s', group.name, group.members, group.pi) # collect group membership entries - member_usernames = {u['sAMAccountName'][0] for u in group.current_ad_users} - missing_usernames + member_usernames = { + u['sAMAccountName'][0] for u in group.current_ad_users + } - missing_usernames missing_members = [ {'username': m['sAMAccountName'][0], 'group': group.name} for m in group.members @@ -599,7 +613,9 @@ def add_new_projects(groupusercollections, errortracker): logger.debug('adding manager status to ProjectUser %s for Project %s', group.pi['sAMAccountName'][0], group.name) try: - manager = group.project.projectuser_set.get(user__username=group.pi['sAMAccountName'][0]) + manager = group.project.projectuser_set.get( + user__username=group.pi['sAMAccountName'][0] + ) except ProjectUser.DoesNotExist: logger.warning('PI %s not found in ProjectUser for Project %s', group.pi['sAMAccountName'][0], group.name) diff --git a/coldfront/plugins/sftocf/tasks.py b/coldfront/plugins/sftocf/tasks.py index d82f58a0f..8982c211a 100644 --- a/coldfront/plugins/sftocf/tasks.py +++ b/coldfront/plugins/sftocf/tasks.py @@ -1,9 +1,4 @@ -from coldfront.plugins.sftocf import utils from django.core import management def pull_sf_push_cf(): management.call_command('pull_sf_push_cf') - - -def pull_resource_data(): - utils.pull_resource_data() diff --git a/coldfront/plugins/sftocf/utils.py b/coldfront/plugins/sftocf/utils.py index c974abb89..8ea4d77cf 100644 --- a/coldfront/plugins/sftocf/utils.py +++ b/coldfront/plugins/sftocf/utils.py @@ -933,52 +933,3 @@ def collect_sf_usage_data(self): entry.pop(item) entries.append(entry) return entries - - -def pull_resource_data(source='rest_api'): - """Pull data from starfish and save to ResourceAttribute objects""" - if source == 'rest_api': - sf = StarFishServer(STARFISH_SERVER) - volumes = sf.get_volume_attributes() - volumes = [ - { - 'name': vol['vol'], - 'attrs': { - 'capacity_tb': vol['total_capacity']/(1024**4), - 'free_tb': vol['free_space']/(1024**4), - 'file_count': vol['number_of_files'], - } - } - for vol in volumes - ] - - elif source == 'redash': - sf = StarFishRedash(STARFISH_SERVER) - volumes = sf.get_vol_stats() - volumes = [ - { - 'name': vol['volume_name'], - 'attrs': { - 'capacity_tb': vol['capacity_TB'], - 'free_tb': vol['free_TB'], - 'file_count': vol['regular_files'], - } - } - for vol in volumes - ] - else: - raise ValueError('source must be "rest_api" or "redash"') - - # collect user and lab counts, allocation sizes for each volume - res_attr_types = ResourceAttributeType.objects.all() - - for volume in volumes: - resource = Resource.objects.get(name__contains=volume['name']) - - for attr_name, attr_val in volume['attrs'].items(): - if attr_val: - attr_type_obj = res_attr_types.get(name=attr_name) - resource.resourceattribute_set.update_or_create( - resource_attribute_type=attr_type_obj, - defaults={'value': attr_val} - ) From 5cb0dc68266d10e2c8b44c6dfc12a4c9f365a99b Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 15 Feb 2024 12:29:07 -0800 Subject: [PATCH 24/28] update add_scheduled_tasks command --- .../utils/management/commands/add_scheduled_tasks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/coldfront/core/utils/management/commands/add_scheduled_tasks.py b/coldfront/core/utils/management/commands/add_scheduled_tasks.py index 3147c089e..71bfc3dc2 100644 --- a/coldfront/core/utils/management/commands/add_scheduled_tasks.py +++ b/coldfront/core/utils/management/commands/add_scheduled_tasks.py @@ -25,9 +25,11 @@ def handle(self, *args, **options): # if plugins are installed, add their tasks kwargs = { "repeats":-1, } plugins_tasks = { - 'fasrc': ['import_quotas', 'id_import_allocations'], - 'sftocf': ['pull_sf_push_cf', 'pull_resource_data'], + 'fasrc': ['import_quotas', 'id_import_allocations', 'pull_resource_data'], + 'sftocf': ['pull_sf_push_cf', 'update_zones'], 'ldap': ['update_group_membership_ldap', 'id_add_projects'], + 'isilon': [], + '': [], 'slurm': ['slurm_sync'], 'xdmod': ['xdmod_usage'], } @@ -40,11 +42,12 @@ def handle(self, *args, **options): schedule(f'coldfront.plugins.{plugin}.tasks.{task}', next_run=date, schedule_type=Schedule.DAILY, + name=task, **kwargs) - if f'coldfront.core.allocation.tasks.send_request_reminder_emails' not in scheduled: + if 'coldfront.core.allocation.tasks.send_request_reminder_emails' not in scheduled: schedule( - f'coldfront.core.allocation.tasks.send_request_reminder_emails', + 'coldfront.core.allocation.tasks.send_request_reminder_emails', next_run=date, schedule_type=Schedule.WEEKLY, **kwargs From 8a460565b5ea1464e58762055b05323c9144aac9 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 15 Feb 2024 12:29:38 -0800 Subject: [PATCH 25/28] update add_scheduled_tasks command --- coldfront/core/utils/management/commands/add_scheduled_tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/coldfront/core/utils/management/commands/add_scheduled_tasks.py b/coldfront/core/utils/management/commands/add_scheduled_tasks.py index 71bfc3dc2..c8f466285 100644 --- a/coldfront/core/utils/management/commands/add_scheduled_tasks.py +++ b/coldfront/core/utils/management/commands/add_scheduled_tasks.py @@ -28,8 +28,6 @@ def handle(self, *args, **options): 'fasrc': ['import_quotas', 'id_import_allocations', 'pull_resource_data'], 'sftocf': ['pull_sf_push_cf', 'update_zones'], 'ldap': ['update_group_membership_ldap', 'id_add_projects'], - 'isilon': [], - '': [], 'slurm': ['slurm_sync'], 'xdmod': ['xdmod_usage'], } From 308b1921747ff569ca18291c31fe1d6a0af4b0e3 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 15 Feb 2024 18:58:28 -0500 Subject: [PATCH 26/28] improve quota modification function and logging, bugfixes --- coldfront/config/logging.py | 3 +- coldfront/config/plugins/isilon.py | 12 +++++ coldfront/core/allocation/views.py | 4 +- .../management/commands/pull_resource_data.py | 8 ++-- coldfront/plugins/isilon/utils.py | 45 ++++++++++++++----- 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/coldfront/config/logging.py b/coldfront/config/logging.py index 2ad433d2c..371d2f359 100644 --- a/coldfront/config/logging.py +++ b/coldfront/config/logging.py @@ -30,7 +30,6 @@ 'handlers': { 'console': { 'class': 'logging.StreamHandler', - 'formatter': 'key-events', }, 'django-q': { 'class': 'logging.handlers.TimedRotatingFileHandler', @@ -60,7 +59,7 @@ 'handlers': ['console', ], }, 'django': { - 'handlers': ['console', 'key-events'], + 'handlers': ['console'], 'level': 'INFO', }, 'django-q': { diff --git a/coldfront/config/plugins/isilon.py b/coldfront/config/plugins/isilon.py index 6b1169bc0..104bf83c7 100644 --- a/coldfront/config/plugins/isilon.py +++ b/coldfront/config/plugins/isilon.py @@ -6,3 +6,15 @@ ISILON_USER = ENV.str('ISILON_USER', '') ISILON_PASS = ENV.str('ISILON_PASS', '') +LOGGING['handlers']['isilon'] = { + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': 'logs/isilon.log', + 'when': 'D', + 'backupCount': 10, # how many backup files to keep + 'formatter': 'default', + 'level': 'DEBUG', +} + +LOGGING['loggers']['coldfront.plugins.isilon'] = { + 'handlers': ['isilon'], +} diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index 0c881cecc..f5e7c6dd1 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -74,7 +74,7 @@ if 'django_q' in settings.INSTALLED_APPS: from django_q.tasks import Task if 'coldfront.plugins.isilon' in settings.INSTALLED_APPS: - from coldfront.plugins.isilon import update_isilon_allocation_quota + from coldfront.plugins.isilon.utils import update_isilon_allocation_quota ALLOCATION_ENABLE_ALLOCATION_RENEWAL = import_from_settings( 'ALLOCATION_ENABLE_ALLOCATION_RENEWAL', True) @@ -1881,7 +1881,7 @@ def post(self, request, *args, **kwargs): messages.error(request, err) return self.redirect_to_detail(pk) - new_quota_value = new_quota['value'] + new_quota_value = int(new_quota['new_value']) plugin = resources_plugins[rtype] if plugin in settings.INSTALLED_APPS: # try to run the thing diff --git a/coldfront/plugins/fasrc/management/commands/pull_resource_data.py b/coldfront/plugins/fasrc/management/commands/pull_resource_data.py index d5d45e7a9..7dee2ffcf 100644 --- a/coldfront/plugins/fasrc/management/commands/pull_resource_data.py +++ b/coldfront/plugins/fasrc/management/commands/pull_resource_data.py @@ -24,7 +24,8 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'source', + '--source', + dest='source', default='rest_api', help='Do not make any changes to Starfish, just print what changes would be slated', ) @@ -77,8 +78,9 @@ def handle(self, *args, **options): } update_resource_attr_types_from_dict(resource, attr_pairs) else: - volume = next([vol for vol in volumes if vol['name'] == resource_name], None) - if not volume: + try: + volume = [v for v in volumes if v['name'] == resource_name][0] + except: logger.error('resource not found in starfish: %s', resource) continue update_resource_attr_types_from_dict(resource, volume['attrs']) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index 434377815..e48eeb5b4 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -33,7 +33,8 @@ def total_space(self): if self._total_space is None: pool_query = self.pools_client.get_storagepool_storagepools( toplevels=True) - self._total_space = pool_query.usage.usable_bytes + pools_bytes = [int(sp.usage.usable_bytes) for sp in pool_query.storagepools] + self._total_space = sum(pools_bytes) return self._total_space @property @@ -53,7 +54,17 @@ def free_space(self): def to_tb(self, bytes_value): return bytes_value / (1024**3) -def update_isilon_allocation_quota(allocation, quota): + def get_quota_from_path(self, path): + current_quota = self.quota_client.list_quota_quotas( + path=path, recurse_path_children=False, recurse_path_parents=False, type='directory') + if len(current_quota.quotas) > 1: + raise Exception(f'more than one quota returned for quota {self.cluster_name}:{path}') + elif len(current_quota.quotas) == 0: + raise Exception(f'no quotas returned for quota {self.cluster_name}:{path}') + return current_quota.quotas[0] + + +def update_isilon_allocation_quota(allocation, new_quota): """Update the quota for an allocation on an isilon cluster Parameters @@ -63,25 +74,35 @@ def update_isilon_allocation_quota(allocation, quota): quota : int """ # make isilon connection to the allocation's resource - isilon_resource = allocation.resources.first().split('/')[0] + isilon_resource = allocation.resources.first().name.split('/')[0] isilon_conn = IsilonConnection(isilon_resource) path = f'/ifs/{allocation.path}' # check if enough space exists on the volume - quota_bytes = quota * 1024**3 + new_quota_bytes = new_quota * 1024**4 unallocated_space = isilon_conn.free_space - allowable_space = unallocated_space * 0.8 - if allowable_space < quota_bytes: + current_quota_obj = isilon_conn.get_quota_from_path(path) + current_quota = current_quota_obj.thresholds.hard + logger.warning("changing allocation %s %s from %s (%s TB) to %s (%s TB)", + allocation.path, allocation, current_quota, allocation.size, new_quota_bytes, new_quota + ) + if unallocated_space < (new_quota_bytes-current_quota): + raise ValueError( + 'ERROR: not enough space on volume to set quota to %s TB for %s' + % (new_quota, allocation) + ) + elif current_quota > new_quota_bytes: raise ValueError( - 'ERROR: not enough space on volume to set quota to %s for %s' - % (quota, allocation) + 'ERROR: cannot automatically shrink the size of allocations at this time. Current size: %s Desired size: %s Allocation: %s' + % (allocation.size, new_quota, allocation) ) try: - isilon_conn.quota_client.update_quota_quota(path=path, threshold_hard=quota) - print(f'SUCCESS: updated quota for {allocation} to {quota}') - logger.info('SUCCESS: updated quota for %s to %s', allocation, quota) + new_quota_obj = {'thresholds': {'hard': new_quota_bytes}} + isilon_conn.quota_client.update_quota_quota(new_quota_obj, current_quota_obj.id) + print(f'SUCCESS: updated quota for {allocation} to {new_quota}') + logger.info('SUCCESS: updated quota for %s to %s', allocation, new_quota) except ApiException as e: - err = f'ERROR: could not update quota for {allocation} to {quota} - {e}' + err = f'ERROR: could not update quota for {allocation} to {new_quota} - {e}' print_log_error(e, err) raise From dcf71bb4bb5fd52bda73dfdedf5bf130299566f4 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 20 Feb 2024 15:26:09 -0500 Subject: [PATCH 27/28] change the size reduction checks --- coldfront/plugins/isilon/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/coldfront/plugins/isilon/utils.py b/coldfront/plugins/isilon/utils.py index e48eeb5b4..24e3e1eb4 100644 --- a/coldfront/plugins/isilon/utils.py +++ b/coldfront/plugins/isilon/utils.py @@ -92,10 +92,13 @@ def update_isilon_allocation_quota(allocation, new_quota): % (new_quota, allocation) ) elif current_quota > new_quota_bytes: - raise ValueError( - 'ERROR: cannot automatically shrink the size of allocations at this time. Current size: %s Desired size: %s Allocation: %s' - % (allocation.size, new_quota, allocation) - ) + current_quota_usage = current_quota_obj.usage.physical + space_needed = new_quota_bytes * .8 + if current_quota_usage > space_needed: + raise ValueError( + 'ERROR: cannot automatically shrink the size of allocations to a quota smaller than 80% of the space in use. Current size: %s Desired size: %s Space used: %s Allocation: %s' + % (allocation.size, new_quota, allocation.usage, allocation) + ) try: new_quota_obj = {'thresholds': {'hard': new_quota_bytes}} isilon_conn.quota_client.update_quota_quota(new_quota_obj, current_quota_obj.id) From fd67a6e26d66dadd057549736546c5fa8cdfc5ca Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 20 Feb 2024 16:13:40 -0500 Subject: [PATCH 28/28] change setting order back --- coldfront/config/settings.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 725217259..2f997c562 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -18,21 +18,21 @@ # ColdFront plugin settings plugin_configs = { - 'PLUGIN_API': 'plugins/api.py', + 'PLUGIN_SLURM': 'plugins/slurm.py', + 'PLUGIN_IQUOTA': 'plugins/iquota.py', + 'PLUGIN_FREEIPA': 'plugins/freeipa.py', + 'PLUGIN_SYSMON': 'plugins/system_montior.py', + 'PLUGIN_XDMOD': 'plugins/xdmod.py', 'PLUGIN_AUTH_OIDC': 'plugins/openid.py', 'PLUGIN_AUTH_LDAP': 'plugins/ldap.py', - 'PLUGIN_LDAP': 'plugins/ldap_fasrc.py', 'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py', + 'PLUGIN_API': 'plugins/api.py', + 'PLUGIN_LDAP': 'plugins/ldap_fasrc.py', + 'PLUGIN_SFTOCF': 'plugins/sftocf.py', 'PLUGIN_FASRC': 'plugins/fasrc.py', - 'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py', - 'PLUGIN_FREEIPA': 'plugins/freeipa.py', 'PLUGIN_IFX': 'plugins/ifx.py', + 'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py', 'PLUGIN_ISILON': 'plugins/isilon.py', - 'PLUGIN_IQUOTA': 'plugins/iquota.py', - 'PLUGIN_SFTOCF': 'plugins/sftocf.py', - 'PLUGIN_SYSMON': 'plugins/system_monitor.py', - 'PLUGIN_SLURM': 'plugins/slurm.py', - 'PLUGIN_XDMOD': 'plugins/xdmod.py', } # This allows plugins to be enabled via environment variables. Can alternatively