Issue166 redcap form status (#43)

* adding completion field to metadata

* added a bunch of print statements

* convert metadata to string then back to json to add field

* metadata from json to string, add field, back to json

* fixed json format

* reading which forms are completed in record set

* adding field to meta in form builder, adding findcompletedforms function to driver

* colors show up

* find form completion for nonlongtudinal studies added

* fixed spacing and added comments

* moved function calls back to top of class due to 'before assignment' error

* deleting methods that were created to find completed forms but are now obsolete

* method find completed forms longitudinal is reformualted for quicker runtime

* method for finding completed forms on longitudinal studies was redone

* removed extra spacing and unneccesary comments and function

* fixed spacing

* fixed spacing

* fixed spacing

* fixed spacing

* modified find completed forms nonlongitudinal to improve runtime

* separated find redcap completion form methods into a new method outside of subrecordgenerationform

* fixed spacing

* removing unnecessary changes

* fix spacing

* removing unnecessary arguement in subrecordselectionform

* reorder arguements to match unit tests

* removed unnecessary arguement from formbuilder constructform

* seperated out add completion field in formbuilder to new function

* adding assert to construct form to assert that completion field was added to metadata

* added 3 unit tests to test button colors, and finding completion fields for long and nonlong

* added payload for testing find_completion_codes_nonlong and modified completion field in one of the response payloads to indicate unverified or complete

* spacing at top

* using .format() for coloring of buttons, and adding icons for color blind

* renaming i,j,l for creating table

* combining codes for nonlong and long studies for method find_completed_form

* deprecated function updated

* rename function

* finding the field for study id in driver config instead of hardcoding

* created more generic add field to form function that can be used beyond this project

* adding json obj to meta instead of string

* removing unecessary arguements

* moving find completion codes to brp

* spacing

* spacing

* new test for method add_new_field_to_form

* reverting changes to original file because values are no longer needed

* added assertions and took away unit tests because methods were moved to brp

* spacing

* removing unused libraries

* using join instead of lambda

ehb_datasources/drivers/redcap/driver.py

@@ -604,7 +604,9 @@ def configure(self, driver_configuration='', *args, **kwargs):
             self.form_event_data = kwargs.pop('form_event_data', None)
             self.form_names = kwargs.pop('form_names', None)
 
-    def subRecordSelectionForm(self, form_url='', *args, **kwargs):
+
+    def subRecordSelectionForm(self, form_url='', redcap_form_complete_codes={}, *args, **kwargs):
+
         '''
         Generates the REDCap data entry table.
 
@@ -629,27 +631,68 @@ def counter(start):
                 yield start
                 start += 1
 
+        # method to identify button color and icon
+        # based on form completion status
+        def get_button_icon(key):
+            all_form_status = redcap_form_complete_codes
+            button_icon=[]
+            try:
+                if all_form_status[key] == 1: # form is unverified
+                    button_icon.append('btn-warning')
+                    button_icon.append('fa-adjust')
+                    return button_icon
+                elif all_form_status[key] == 2: # form is complete
+                    button_icon.append('btn-success')
+                    button_icon.append('fa-circle')
+                    return button_icon
+            except: # form is incomplete
+                button_icon.append('btn-primary')
+                button_icon.append('fa-circle-o')
+                return button_icon
+
         if self.form_names:
             # The project is not longitudinal
-            def makeRow(fn, i):
-                row = '<tr><td>' + reduce(
-                    lambda x,
-                    y: x + ' ' + y.capitalize(),
-                    fn.split('_'), '') + '</td>'
+            def makeRow(form_name, form_index):
+                key = str(form_index)
+                row = '<tr><td>' + " ".join([fn.capitalize() for fn in form_name.split('_')]) + '</td>'
                 return row + ('<td><button data-toggle="modal"' +
-                              ' data-backdrop="static" data-keyboard="false"' +
-                              ' href="#pleaseWaitModal" class="btn btn-small' +
-                              ' btn-primary" onclick="location.href=\'' +
-                              form_url + str(i) + '/\'">Edit</button></td>')
-
+                            ' data-backdrop="static" data-keyboard="false"' +
+                            ' href="#pleaseWaitModal" class="btn btn-small ' + '{button_icon[0]}'
+                            ' " onclick="location.href=\'' + form_url + key +
+                            '/\'">Edit <i class="fa ' +'{button_icon[1]}' +
+                            '"></i></button></td>').format(button_icon=get_button_icon(key))
             form = ('<table class="table table-bordered table-striped ' +
                     'table-condensed"><tr><th>Data Form</th><th></th></tr>')
             count = counter(0)
-            rows = [makeRow(fn, next(count)) for fn in self.form_names]
+            rows = [makeRow(form_name, next(count)) for form_name in self.form_names]
             form += ''.join(rows) + '</table>'
             return form
         else:
             # The project is longitudinal
+            def make_td(form_index, event_index, form_exists):
+                key = str(form_index) + "_" + str(event_index)
+                if form_exists:
+                    return ('<td><button data-toggle="modal"' +
+                            'data-backdrop="static" data-keyboard="false" ' +
+                            'href="#pleaseWaitModal" class="btn btn-small ' +
+                            '{button_icon[0]}' + '" onclick="location.href=\'' +
+                            form_url + key + '/\'">Edit <i class="fa ' +
+                            '{button_icon[1]}' + '"></i></button></td>').format(button_icon=get_button_icon(key))
+                else:
+                    return '<td></td>'
+
+            def make_trs(form_index, form_list):
+                count = counter(0)
+                if len(form_list) > 1:
+                    form_name_cell = '<tr><td>' + " ".join([fn.capitalize() for fn in form_list[0].split('_')]) + '</td>'
+                    edit_button_cells = reduce( lambda x,y: x + make_td(form_index, next(count), y),
+                                        self.form_data[form_list[0]], '') + '</tr>'
+                    return form_name_cell + edit_button_cells + make_trs(form_index + 1, form_list[1: len(form_list)])
+                else:
+                    form_name_cell = '<tr><td>' + " ".join([fn.capitalize() for fn in form_list[0].split('_')]) + '</td>'
+                    edit_button_cells = reduce( lambda x,y: x + make_td(form_index, next(count), y),
+                                        self.form_data[form_list[0]], '')
+                    return form_name_cell + edit_button_cells
             number_of_events = str(len(self.event_labels))
             form = ('<table class="table table-bordered table-striped' +
                     'table-condensed"><tr><th rowspan="2">' +
@@ -660,39 +703,6 @@ def makeRow(fn, i):
                 y: x + '<td>' + y + '</td>',
                 self.event_labels,
                 '') + '</tr>'
-
-            def make_td(i, j, l):
-                if l:
-                    return ('<td><button data-toggle="modal"' +
-                            'data-backdrop="static" data-keyboard="false" ' +
-                            'href="#pleaseWaitModal" class="btn btn-small ' +
-                            'btn-primary" onclick="location.href=\'' +
-                            form_url +
-                            str(i) + '_' + str(j) + '/\'">Edit</button></td>')
-                else:
-                    return '<td></td>'
-
-            def make_trs(i, l):
-                count = counter(0)
-                if len(l) > 1:
-                    return '<tr><td>' + reduce(
-                        lambda x,
-                        y: x + ' ' + y.capitalize(),
-                        l[0].split('_'), '') + '</td>' + reduce(
-                        lambda x,
-                        y: x + make_td(i, next(count), y),
-                        self.form_data[l[0]],
-                        '') + '</tr>' + make_trs(i + 1, l[1: len(l)])
-                else:
-                    return '<tr><td>' + reduce(
-                        lambda x,
-                        y: x + ' ' + y.capitalize(),
-                        l[0].split('_'), '') + '</td>' + reduce(
-                        lambda x,
-                        y: x + make_td(i, next(count), y),
-                        self.form_data[l[0]],
-                        '')
-
             form += make_trs(0, self.form_data_ordered) + '</table>'
             return form
 
@@ -712,7 +722,6 @@ def subRecordForm(self, external_record, form_spec='', *args, **kwargs):
 
         The form and event numbers are mapped to form names and event names in
         the order they were provided in the call to configure
-
         If the REDCap project is not longitudinal (i.e. Survey or Data Forms
         Classic) the event number is not required and will be ignored if
         included

ehb_datasources/drivers/redcap/formBuilderJson.py

@@ -20,6 +20,36 @@ class redcapTemplate(Template):
 
 class FormBuilderJson(object):
 
+    # this method can be used to add a field not already defined in meta data to all forms
+    def add_new_field_to_form (self, meta, field_name="", form_name="", field_type="", field_label="", select_choices_or_calculations="",
+        section_header="", field_note="", text_validation_type_or_show_slider_number="",
+        text_validation_min="",text_validation_max="",identifier="", branching_logic="",
+        required_field="", custom_alignment="", question_number="", matrix_group_name="",
+        matrix_ranking="", field_annotation=""):
+        new_field = {}
+        new_field["field_name"] = field_name
+        new_field["form_name"] = form_name
+        new_field["section_header"] = section_header
+        new_field["field_type"] = field_type
+        new_field ["field_label"] = field_label
+        new_field["select_choices_or_calculations"]= select_choices_or_calculations
+        new_field["field_note"] = field_note
+        new_field["text_validation_type_or_show_slider_number"] = text_validation_type_or_show_slider_number
+        new_field["text_validation_min"] = text_validation_min
+        new_field["text_validation_max"] = text_validation_max
+        new_field["identifier"] = identifier
+        new_field["branching_logic"] = branching_logic
+        new_field["required_field"] = required_field
+        new_field["custom_alignment"] = custom_alignment
+        new_field["question_number"] = question_number
+        new_field["matrix_group_name"] = matrix_group_name
+        new_field["matrix_ranking"]= matrix_ranking
+        new_field["field_annotation"] = field_annotation
+        new_field = json.dumps (new_field)
+        new_field = json.loads(new_field)
+        meta.append(new_field)
+        return meta
+
     def construct_form(self, meta, record_set, form_name, record_id,
                        event_num=None, unique_event_names=None,
                        event_labels=None, session=None, record_id_field=None):
@@ -64,6 +94,14 @@ def construct_form(self, meta, record_set, form_name, record_id,
                 return '''
                     <div class="alert alert-danger"><center><span>There was an error retrieving this record from REDCap</span></center></div>
                 '''
+
+        # construct the field name for adding completion status to redcap forms
+        completion_field_name = form_name + "_complete"
+        # add completion field to all redcap forms
+        meta = self.add_new_field_to_form (meta, field_name=completion_field_name, form_name=form_name,
+            field_type="dropdown", field_label="Form Completion Status", select_choices_or_calculations="0, Incomplete | 1, Unverified | 2, Complete",
+            section_header="Form Status", required_field="y")
+
         form_fields = [
             item for item in meta if item.get("form_name") == form_name
         ]

ehb_datasources/tests/unit_tests/test_form_builder.py

@@ -20,6 +20,7 @@ def test_construct_form(form_builder, redcap_metadata_json, redcap_record_json):
     assert '<input type="text" value="100" name="height" class="field_input" id="input_height"  />' in form
     assert '<input type="text" value="20" name="weight" class="field_input" id="input_weight"  />' in form
     assert '<textarea rows="5" cols="20" name="comments" class="field_input" >Test Data2</textarea>' in form
+    assert 'Form Completion Status' in form
 
 def test_construct_form2_branch_logic_functions(form_builder, redcap_metadata_json2, redcap_record_json2):
     form = form_builder.construct_form(
@@ -84,3 +85,9 @@ def test_construct_form_bad_redcap_record(form_builder, redcap_metadata_json2):
         'study_id'
     )
     assert 'There was an error retrieving this record from REDCap' in form
+
+def test_add_new_field_to_form(form_builder, redcap_metadata_json):
+    metadata_json = json.loads(redcap_metadata_json.decode('utf-8'))
+    new_field = form_builder.add_new_field_to_form(metadata_json, field_name="test_new_field")
+    last_index = len(new_field)-1
+    assert 'test_new_field' in new_field[last_index]['field_name']

ehb_datasources/tests/unit_tests/test_redcap_driver.py

@@ -788,3 +788,25 @@ def test_write_records_badresp(mocker, driver, redcap_payload):
         {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'text/xml'},
         'content=record&data=%3Crecords%3E%3Citem%3E%3Cstudy_id%3E%3C%21%5BCDATA%5B0GUQDBCDE0EAWN9Q%3A8LAG76CHO%5D%5D%3E%3C%2Fstudy_id%3E%3Credcap_event_name%3E%3C%21%5BCDATA%5Bvisit_arm_1%5D%5D%3E%3C%2Fredcap_event_name%3E%3Ccolonoscopy_date%3E%3C%21%5BCDATA%5B2016-08-31%5D%5D%3E%3C%2Fcolonoscopy_date%3E%3Cgeneral_ibd%3E%3C%21%5BCDATA%5B2016-08-31%5D%5D%3E%3C%2Fgeneral_ibd%3E%3Ctransferrin_b%3E%3C%21%5BCDATA%5B102%5D%5D%3E%3C%2Ftransferrin_b%3E%3Cmeds___2%3E%3C%21%5BCDATA%5B0%5D%5D%3E%3C%2Fmeds___2%3E%3Cmeal_date%3E%3C%21%5BCDATA%5B2016-08-31%5D%5D%3E%3C%2Fmeal_date%3E%3Culcerative_colitis%3E%3C%21%5BCDATA%5B2016-08-31%5D%5D%3E%3C%2Fulcerative_colitis%3E%3Cmeds___1%3E%3C%21%5BCDATA%5B1%5D%5D%3E%3C%2Fmeds___1%3E%3Ccomments%3E%3C%21%5BCDATA%5BTest+Data%5D%5D%3E%3C%2Fcomments%3E%3Cweight%3E%3C%21%5BCDATA%5B20%5D%5D%3E%3C%2Fweight%3E%3Cchrons%3E%3C%21%5BCDATA%5B2016-08-31%5D%5D%3E%3C%2Fchrons%3E%3Cchol_b%3E%3C%21%5BCDATA%5B101%5D%5D%3E%3C%2Fchol_b%3E%3Ccolonoscopy%3E%3C%21%5BCDATA%5B0%5D%5D%3E%3C%2Fcolonoscopy%3E%3Cprealb_b%3E%3C%21%5BCDATA%5B19%5D%5D%3E%3C%2Fprealb_b%3E%3Cheight%3E%3C%21%5BCDATA%5B100%5D%5D%3E%3C%2Fheight%3E%3Ccreat_b%3E%3C%21%5BCDATA%5B0.6%5D%5D%3E%3C%2Fcreat_b%3E%3Cibd_flag%3E%3C%21%5BCDATA%5B1%5D%5D%3E%3C%2Fibd_flag%3E%3Cmeds___5%3E%3C%21%5BCDATA%5B0%5D%5D%3E%3C%2Fmeds___5%3E%3Cmeds___4%3E%3C%21%5BCDATA%5B0%5D%5D%3E%3C%2Fmeds___4%3E%3Cmeds___3%3E%3C%21%5BCDATA%5B0%5D%5D%3E%3C%2Fmeds___3%3E%3C%2Fitem%3E%3C%2Frecords%3E&format=xml&overwriteBehavior=overwrite&token=foo&type=flat'
     )
+
+
+def test_srsf_redcap_completion_codes(mocker, driver, driver_configuration_long, redcap_metadata_json, redcap_record_json):
+    # Mocks
+    # Metadata request
+    driver.meta = mocker.MagicMock(return_value=redcap_metadata_json)
+    driver.configure(driver_configuration_long)
+    # Record Request
+    MockREDCapResponse = mocker.MagicMock(
+        spec=HTTPResponse,
+        status=200)
+    MockREDCapResponse.read = mocker.MagicMock(return_value=redcap_record_json)
+    driver.POST = mocker.MagicMock(return_value=MockREDCapResponse)
+
+    redcap_form_complete_codes = {}
+    redcap_form_complete_codes[(str(1)+"_"+str(3))] = 2
+    redcap_form_complete_codes[(str(0)+"_"+str(0))] = 1
+    form = driver.subRecordSelectionForm(form_url='/test/', redcap_form_complete_codes=redcap_form_complete_codes)
+    assert 'btn-success' in form
+    assert 'fa-circle' in form
+    assert 'btn-warning' in form
+    assert 'fa-adjust' in form