Merge pull request #278 from fasrc/cp_isilon

Isilon plugin

coldfront/config/logging.py

@@ -30,7 +30,6 @@
     'handlers': {
         'console': {
             'class': 'logging.StreamHandler',
-            'formatter': 'key-events',
         },
         'django-q': {
             'class': 'logging.handlers.TimedRotatingFileHandler',
@@ -60,7 +59,7 @@
             'handlers': ['console', ],
         },
         'django': {
-            'handlers': ['console', 'key-events'],
+            'handlers': ['console'],
             'level': 'INFO',
         },
         'django-q': {

coldfront/config/plugins/isilon.py

@@ -0,0 +1,20 @@
+from coldfront.config.env import ENV
+from coldfront.config.logging import LOGGING
+from coldfront.config.base import INSTALLED_APPS
+
+INSTALLED_APPS += [ 'coldfront.plugins.isilon' ]
+ISILON_USER = ENV.str('ISILON_USER', '')
+ISILON_PASS = ENV.str('ISILON_PASS', '')
+
+LOGGING['handlers']['isilon'] = {
+    'class': 'logging.handlers.TimedRotatingFileHandler',
+    'filename': 'logs/isilon.log',
+    'when': 'D',
+    'backupCount': 10, # how many backup files to keep
+    'formatter': 'default',
+    'level': 'DEBUG',
+}
+
+LOGGING['loggers']['coldfront.plugins.isilon'] = {
+    'handlers': ['isilon'],
+}

coldfront/config/settings.py

@@ -32,6 +32,7 @@
     'PLUGIN_FASRC': 'plugins/fasrc.py',
     'PLUGIN_IFX': 'plugins/ifx.py',
     'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py',
+    'PLUGIN_ISILON': 'plugins/isilon.py',
 }
 
 # This allows plugins to be enabled via environment variables. Can alternatively

coldfront/core/allocation/admin.py

@@ -29,7 +29,7 @@ class AllocationStatusChoiceAdmin(admin.ModelAdmin):
 class AllocationUserInline(admin.TabularInline):
     model = AllocationUser
     extra = 0
-    fields = ('id', 'user', 'status', 'usage',)
+    fields = ('id', 'user', 'status', 'usage', 'usage_bytes', 'unit')
 
 
 class AllocationAttributeInline(admin.TabularInline):

coldfront/core/allocation/forms.py

@@ -40,10 +40,10 @@ def validate(self, value):
                     "Input must consist only of digits (or x'es) and dashes."
                 )
             if len(digits_only(value)) != 33:
-                raise ValidationError("Input must contain exactly 33 digits.")
+                raise ValidationError('Input must contain exactly 33 digits.')
             if 'x' in digits_only(value)[:8]+digits_only(value)[12:]:
                 raise ValidationError(
-                    "xes are only allowed in place of the product code (the third grouping of characters in the code)"
+                    'xes are only allowed in place of the product code (the third grouping of characters in the code)'
                 )
 
     def clean(self, value):
@@ -81,7 +81,7 @@ class AllocationForm(forms.Form):
     )
 
     expense_code = ExpenseCodeField(
-        label="...or add a new 33 digit expense code manually here.", required=False
+        label='...or add a new 33 digit expense code manually here.', required=False
     )
 
     tier = forms.ModelChoiceField(
@@ -110,7 +110,7 @@ def __init__(self, request_user, project_pk,  *args, **kwargs):
         project_obj = get_object_or_404(Project, pk=project_pk)
         self.fields['tier'].queryset = get_user_resources(request_user).filter(
             resource_type__name='Storage Tier'
-        ).order_by(Lower("name"))
+        ).order_by(Lower('name'))
         existing_expense_codes = [(None, '------')] + [
             (a.code, f'{a.code} ({a.name})') for a in Account.objects.filter(
                 userproductaccount__is_valid=1,
@@ -120,28 +120,28 @@ def __init__(self, request_user, project_pk,  *args, **kwargs):
         self.fields['existing_expense_codes'].choices = existing_expense_codes
         user_query_set = project_obj.projectuser_set.select_related('user').filter(
             status__name__in=['Active', ]
-        ).order_by("user__username").exclude(user=project_obj.pi)
+        ).order_by('user__username').exclude(user=project_obj.pi)
         # if user_query_set:
         #     self.fields['users'].choices = ((user.user.username, "%s %s (%s)" % (
-        #         user.user.first_name, user.user.last_name, user.user.username)) for user in user_query_set)
+        #         u.user.first_name, u.user.last_name, u.user.username)) for u in user_query_set)
         #     self.fields['users'].help_text = '<br/>Select users in your project to add to this allocation.'
         # else:
         #     self.fields['users'].widget = forms.HiddenInput()
 
     def clean(self):
         cleaned_data = super().clean()
         # Remove all dashes from the input string to count the number of digits
-        expense_code = cleaned_data.get("expense_code")
-        existing_expense_codes = cleaned_data.get("existing_expense_codes")
+        expense_code = cleaned_data.get('expense_code')
+        existing_expense_codes = cleaned_data.get('existing_expense_codes')
         trues = sum(x for x in [
             (expense_code not in ['', '------']),
             (existing_expense_codes not in ['', '------']),
         ])
         digits_only = lambda v: re.sub(r'[^0-9xX]', '', v)
         if trues != 1:
             self.add_error(
-                "existing_expense_codes",
-                "You must either select an existing expense code or manually enter a new one."
+                'existing_expense_codes',
+                'You must either select an existing expense code or manually enter a new one.'
             )
 
         elif expense_code and expense_code != '------':
@@ -169,7 +169,7 @@ class AllocationUpdateForm(forms.Form):
         label='Resource', queryset=Resource.objects.all(), required=False
     )
     status = forms.ModelChoiceField(
-        queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")), empty_label=None)
+        queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')), empty_label=None)
     start_date = forms.DateField(
         label='Start Date',
         widget=forms.DateInput(attrs={'class': 'datepicker'}),
@@ -204,8 +204,8 @@ def __init__(self, *args, **kwargs):
 
     def clean(self):
         cleaned_data = super().clean()
-        start_date = cleaned_data.get("start_date")
-        end_date = cleaned_data.get("end_date")
+        start_date = cleaned_data.get('start_date')
+        end_date = cleaned_data.get('end_date')
         if start_date and end_date and end_date < start_date:
             raise forms.ValidationError('End date cannot be less than start date')
         return cleaned_data
@@ -214,7 +214,7 @@ def clean(self):
 class AllocationInvoiceUpdateForm(forms.Form):
     status = forms.ModelChoiceField(queryset=AllocationStatusChoice.objects.filter(
         name__in=['Payment Pending', 'Payment Requested', 'Payment Declined', 'Paid']
-    ).order_by(Lower("name")), empty_label=None)
+    ).order_by(Lower('name')), empty_label=None)
 
 
 class AllocationAddUserForm(forms.Form):
@@ -250,16 +250,16 @@ class AllocationSearchForm(forms.Form):
     username = forms.CharField(label='Username', max_length=100, required=False)
     resource_type = forms.ModelChoiceField(
         label='Resource Type',
-        queryset=ResourceType.objects.all().order_by(Lower("name")),
+        queryset=ResourceType.objects.all().order_by(Lower('name')),
         required=False)
     resource_name = forms.ModelMultipleChoiceField(
         label='Resource Name',
         queryset=Resource.objects.filter(
-            is_allocatable=True).order_by(Lower("name")),
+            is_allocatable=True).order_by(Lower('name')),
         required=False)
     allocation_attribute_name = forms.ModelChoiceField(
         label='Allocation Attribute Name',
-        queryset=AllocationAttributeType.objects.all().order_by(Lower("name")),
+        queryset=AllocationAttributeType.objects.all().order_by(Lower('name')),
         required=False)
     allocation_attribute_value = forms.CharField(
         label='Allocation Attribute Value', max_length=100, required=False)
@@ -273,7 +273,7 @@ class AllocationSearchForm(forms.Form):
         required=False)
     status = forms.ModelMultipleChoiceField(
         widget=forms.CheckboxSelectMultiple,
-        queryset=AllocationStatusChoice.objects.all().order_by(Lower("name")),
+        queryset=AllocationStatusChoice.objects.all().order_by(Lower('name')),
         required=False)
     show_all_allocations = forms.BooleanField(initial=False, required=False)
 
@@ -323,7 +323,7 @@ def __init__(self, *args, **kwargs):
     def clean(self):
         cleaned_data = super().clean()
 
-        if cleaned_data.get('new_value') != "":
+        if cleaned_data.get('new_value') != '':
             allocation_attribute = AllocationAttribute.objects.get(pk=cleaned_data.get('pk'))
             allocation_attribute.value = cleaned_data.get('new_value')
             allocation_attribute.clean()
@@ -351,10 +351,10 @@ def clean(self):
 
 class AllocationChangeForm(forms.Form):
     EXTENSION_CHOICES = [
-        (0, "No Extension")
+        (0, 'No Extension')
     ]
     for choice in ALLOCATION_CHANGE_REQUEST_EXTENSION_DAYS:
-        EXTENSION_CHOICES.append((choice, f"{choice} days"))
+        EXTENSION_CHOICES.append((choice, f'{choice} days'))
 
     end_date_extension = forms.TypedChoiceField(
         label='Request End Date Extension',
@@ -378,7 +378,26 @@ class AllocationChangeNoteForm(forms.Form):
             label='Notes',
             required=False,
             widget=forms.Textarea,
-            help_text="Leave any feedback about the allocation change request.")
+            help_text='Leave any feedback about the allocation change request.')
+
+
+ALLOCATION_AUTOUPDATE_OPTIONS = [
+    ('1', 'I have already manually modified the allocation.'),
+    ('2', 'I would like Coldfront to modify the allocation for me. If Coldfront experiences any issues with the modification process, I understand that I will need to modify the allocation manually instead.'),
+]
+
+class AllocationAutoUpdateForm(forms.Form):
+    sheetcheck = forms.BooleanField(
+        label='I have ensured that enough space is available on this resource.'
+    )
+    auto_update_opts = forms.ChoiceField(
+        label='How will this allocation be modified?',
+        required=True,
+        widget=forms.RadioSelect,
+        choices=ALLOCATION_AUTOUPDATE_OPTIONS,
+    )
+
+
 
 class AllocationAttributeCreateForm(forms.ModelForm):
     class Meta:

coldfront/core/allocation/templates/allocation/allocation_change_detail.html

@@ -197,8 +197,9 @@ <h3 class="d-inline"><i class="fas fa-info-circle" aria-hidden="true"></i> Actio
       </div>
       <div class="card-body">
 
-
-            {% csrf_token %}
+          {% csrf_token %}
+            {{autoupdate_form.sheetcheck | as_crispy_field}}
+            {{autoupdate_form.auto_update_opts | as_crispy_field}}
             {{note_form.notes | as_crispy_field}}
             <div style="float: right;">
               {% if allocation_change.status.name == 'Pending' %}
@@ -209,7 +210,6 @@ <h3 class="d-inline"><i class="fas fa-info-circle" aria-hidden="true"></i> Actio
                 <i class="fas fa-sync" aria-hidden="true"></i> Update
               </button>
             </div>
-        </div>
       </div>
     </div>
   {% endif %}

coldfront/core/allocation/views.py

@@ -35,6 +35,7 @@
                                              AllocationAttributeChangeForm,
                                              AllocationAttributeUpdateForm,
                                              AllocationForm,
+                                             AllocationAutoUpdateForm,
                                              AllocationInvoiceNoteDeleteForm,
                                              AllocationInvoiceUpdateForm,
                                              AllocationRemoveUserForm,
@@ -72,6 +73,8 @@
     from ifxbilling.models import Account, UserProductAccount
 if 'django_q' in settings.INSTALLED_APPS:
     from django_q.tasks import Task
+if 'coldfront.plugins.isilon' in settings.INSTALLED_APPS:
+    from coldfront.plugins.isilon.utils import update_isilon_allocation_quota
 
 ALLOCATION_ENABLE_ALLOCATION_RENEWAL = import_from_settings(
     'ALLOCATION_ENABLE_ALLOCATION_RENEWAL', True)
@@ -1718,9 +1721,12 @@ def get(self, request, *args, **kwargs):
             initial={'notes': allocation_change_obj.notes}
         )
 
+        autoupdate_form = AllocationAutoUpdateForm()
+
         context = self.get_context_data()
 
         context['allocation_change_form'] = allocation_change_form
+        context['autoupdate_form'] = autoupdate_form
         context['note_form'] = note_form
         return render(request, self.template_name, context)
 
@@ -1765,7 +1771,10 @@ def post(self, request, *args, **kwargs):
             context['allocation_change_form'] = allocation_change_form
             return render(request, self.template_name, context)
 
+
         action = request.POST.get('action')
+
+        autoupdate_form = AllocationAutoUpdateForm(request.POST)
         if action not in ['update', 'approve', 'deny']:
             return HttpResponseBadRequest('Invalid request')
 
@@ -1845,9 +1854,54 @@ def post(self, request, *args, **kwargs):
                 alloc_change_obj.allocation.save()
 
             if attrs_to_change:
-                attr_changes = (
-                    alloc_change_obj.allocationattributechangerequest_set.all()
-                )
+                attr_changes = alloc_change_obj.allocationattributechangerequest_set.all()
+
+                autoupdate_choice = autoupdate_form.data.get('auto_update_opts')
+                if autoupdate_choice == '2':
+                    # check resource type, see if appropriate plugin is available
+                    resource = alloc_change_obj.allocation.resources.first().name
+                    resources_plugins = {
+                        'isilon': 'coldfront.plugins.isilon',
+                        # 'lfs': 'coldfront.plugins.lustre',
+                    }
+                    rtype = next((k for k in resources_plugins), None)
+                    if not rtype:
+                        err = ('You cannot auto-update non-isilon resources at this '
+                            'time. Please manually update the resource before '
+                            'approving this change request.')
+                        messages.error(request, err)
+                        return self.redirect_to_detail(pk)
+                    # get new quota value
+                    new_quota = next((
+                        a for a in attrs_to_change if a['name'] == 'Storage Quota (TB)'), None)
+                    if not new_quota:
+                        err = ('You can only auto-update resource quotas at this '
+                            'time. Please manually update the resource before '
+                            'approving this change request.')
+                        messages.error(request, err)
+                        return self.redirect_to_detail(pk)
+
+                    new_quota_value = int(new_quota['new_value'])
+                    plugin = resources_plugins[rtype]
+                    if plugin in settings.INSTALLED_APPS:
+                        # try to run the thing
+                        try:
+                            update_isilon_allocation_quota(
+                                alloc_change_obj.allocation, new_quota_value
+                            )
+                        except Exception as e:
+                            err = ("An error was encountered while auto-updating"
+                                "the allocation quota. Please contact Coldfront "
+                                "administration and/or manually update the allocation.")
+                            messages.error(request, err)
+                            return self.redirect_to_detail(pk)
+                    else:
+                        err = ("There is an issue with the configuration of "
+                            "Coldfront's auto-updating capabilities. Please contact Coldfront "
+                            "administration and/or manually update the allocation.")
+                        messages.error(request, err)
+                        return self.redirect_to_detail(pk)
+
                 for attribute_change in attr_changes:
                     new_value = attribute_change.new_value
                     attribute_change.allocation_attribute.value = new_value

coldfront/core/project/views.py

@@ -163,7 +163,7 @@ def get_context_data(self, **kwargs):
                     )
                 )
             except ValueError:
-                err = "Allocation attribute '{}' is not an int but has a usage".format(
+                err = "Project attribute '{}' is not an int but has a usage".format(
                     attribute.allocation_attribute_type.name
                 )
                 logger.error(err)
@@ -178,9 +178,7 @@ def get_context_data(self, **kwargs):
 
         context['mailto'] = 'mailto:' + ','.join([u.user.email for u in project_users])
 
-        allocations = Allocation.objects.prefetch_related('resources').filter(
-            Q(project=self.object)
-        )
+        allocations = self.object.allocation_set.prefetch_related('resources')
         allocation_history_records = self.return_status_change_records(allocations)
 
         if not self.request.user.is_superuser and not self.request.user.has_perm(

coldfront/core/utils/management/commands/add_scheduled_tasks.py

@@ -25,8 +25,8 @@ def handle(self, *args, **options):
         # if plugins are installed, add their tasks
         kwargs = {  "repeats":-1, }
         plugins_tasks = {
-            'fasrc': ['import_quotas', 'id_import_allocations'],
-            'sftocf': ['pull_sf_push_cf', 'pull_resource_data'],
+            'fasrc': ['import_quotas', 'id_import_allocations', 'pull_resource_data'],
+            'sftocf': ['pull_sf_push_cf', 'update_zones'],
             'ldap': ['update_group_membership_ldap', 'id_add_projects'],
             'slurm': ['slurm_sync'],
             'xdmod': ['xdmod_usage'],
@@ -40,11 +40,12 @@ def handle(self, *args, **options):
                         schedule(f'coldfront.plugins.{plugin}.tasks.{task}',
                             next_run=date,
                             schedule_type=Schedule.DAILY,
+                            name=task,
                             **kwargs)
 
-        if f'coldfront.core.allocation.tasks.send_request_reminder_emails' not in scheduled:
+        if 'coldfront.core.allocation.tasks.send_request_reminder_emails' not in scheduled:
             schedule(
-                f'coldfront.core.allocation.tasks.send_request_reminder_emails',
+                'coldfront.core.allocation.tasks.send_request_reminder_emails',
                 next_run=date,
                 schedule_type=Schedule.WEEKLY,
                 **kwargs