diff --git a/seed/models/column_mapping_profiles.py b/seed/models/column_mapping_profiles.py index 092f606f37..d52d18d6c3 100644 --- a/seed/models/column_mapping_profiles.py +++ b/seed/models/column_mapping_profiles.py @@ -81,6 +81,10 @@ def create_from_file( "to_table_name": row[2], "to_field": row[3], } + try: + data["is_omitted"] = "True" if row[4].lower().strip() == "true" else "False" + except IndexError: + data["is_omitted"] = "False" mappings.append(data) else: raise Exception(f"Mapping file does not exist: {filename}") diff --git a/seed/models/columns.py b/seed/models/columns.py index c2c744ffcb..cf5d82fd5f 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -182,6 +182,23 @@ class Column(models.Model): "PointField": "geometry", } + DB_TYPES = { + "number": "float", + "float": "float", + "integer": "integer", + "string": "string", + "geometry": "geometry", + "datetime": "datetime", + "date": "date", + "boolean": "boolean", + "area": "float", + "eui": "float", + "ghg": "float", + "ghg_intensity": "float", + "wui": "float", + "water_use": "float", + } + DATA_TYPE_PARSERS: dict[str, Callable] = { "number": lambda v: float(v.replace(",", "") if isinstance(v, str) else v), "float": lambda v: float(v.replace(",", "") if isinstance(v, str) else v), @@ -1010,9 +1027,9 @@ def create_mappings_from_file(filename, organization, user, import_file_id=None) "from_field": row[0], "to_table_name": row[1], "to_field": row[2], - # "to_display_name": row[3], - # "to_data_type": row[4], - # "to_unit_type": row[5], + "to_display_name": row[3], + "to_data_type": row[4], + "to_unit_type": row[5], } mappings.append(data) else: @@ -1059,13 +1076,13 @@ def create_mappings(mappings, organization, user, import_file_id=None): 'to_field': 'gross_floor_area', 'to_field_display_name': 'Gross Floor Area', 'to_table_name': 'PropertyState', + 'to_data_type': 'string', # an internal data type mapping } ] """ # initialize a cache to store the mappings cache_column_mapping = [] - # Take the existing object and return the same object with the db column objects added to # the dictionary (to_column_object and from_column_object) mappings = Column._column_fields_to_columns(mappings, organization, user) @@ -1101,7 +1118,6 @@ def create_mappings(mappings, organization, user, import_file_id=None): column_mapping.user = user column_mapping.save() - cache_column_mapping.append( { "from_field": mapping["from_field"], @@ -1176,6 +1192,10 @@ def _column_fields_to_columns(fields, organization, user): "table_name": "" if is_ah_data else field["to_table_name"], "is_extra_data": is_extra_data, } + # Only compare against data type if it is provided && the column is an extra data column + if ("to_data_type" in field) and (is_extra_data): + to_col_params["data_type"] = field["to_data_type"] + if is_root_user or is_ah_data: to_org_col, _ = Column.objects.get_or_create(**to_col_params) else: @@ -1312,28 +1332,10 @@ def retrieve_db_types(): """ columns = copy.deepcopy(Column.DATABASE_COLUMNS) - # TODO: There seem to be lots of these lists floating around. We should consolidate them. - MAP_TYPES = { - "number": "float", - "float": "float", - "integer": "integer", - "string": "string", - "geometry": "geometry", - "datetime": "datetime", - "date": "date", - "boolean": "boolean", - "area": "float", - "eui": "float", - "ghg": "float", - "ghg_intensity": "float", - "wui": "float", - "water_use": "float", - } - types = OrderedDict() for c in columns: try: - types[c["column_name"]] = MAP_TYPES[c["data_type"]] + types[c["column_name"]] = Column.DB_TYPES[c["data_type"]] except KeyError: _log.error("could not find data_type for %s" % c) types[c["column_name"]] = "" diff --git a/seed/static/seed/js/controllers/column_mappings_controller.js b/seed/static/seed/js/controllers/column_mappings_controller.js index a38071b1fa..a5f4db9899 100644 --- a/seed/static/seed/js/controllers/column_mappings_controller.js +++ b/seed/static/seed/js/controllers/column_mappings_controller.js @@ -423,11 +423,21 @@ angular.module('SEED.controller.column_mappings', []).controller('column_mapping return Boolean(_.find(_.values(grouped_by_from_field), (group) => group.length > 1)); }; + $scope.save_button_tooltip = () => { + if (!$scope.changes_possible) return 'No changes'; + if ($scope.header_duplicates_present()) return 'Duplicate headers present'; + if ($scope.empty_units_present()) return 'Empty units present'; + if ($scope.current_profile.profile_type === COLUMN_MAPPING_PROFILE_TYPE_BUILDINGSYNC_DEFAULT) return 'This profile cannot be changed'; + if (!$scope.profile_action_ok('update')) return 'Unknown profile type'; + return 'Save'; + }; + $scope.empty_units_present = () => Boolean( _.find($scope.current_profile.mappings, (field) => { const has_units = $scope.is_pint_column(field); return field.to_table_name === 'PropertyState' && field.from_units === null && has_units; }) + ); $scope.profile_action_ok = (action) => { diff --git a/seed/static/seed/js/controllers/mapping_controller.js b/seed/static/seed/js/controllers/mapping_controller.js index 8d75e5efc6..025891ed41 100644 --- a/seed/static/seed/js/controllers/mapping_controller.js +++ b/seed/static/seed/js/controllers/mapping_controller.js @@ -74,6 +74,20 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ COLUMN_MAPPING_PROFILE_TYPE_BUILDINGSYNC_CUSTOM, derived_columns_payload ) { + $scope.data_types = [ + { id: 'None', label: 'None' }, + { id: 'number', label: $translate.instant('Number') }, + { id: 'integer', label: $translate.instant('Integer') }, + { id: 'string', label: $translate.instant('Text') }, + { id: 'datetime', label: $translate.instant('Datetime') }, + { id: 'date', label: $translate.instant('Date') }, + { id: 'boolean', label: $translate.instant('Boolean') }, + { id: 'area', label: $translate.instant('Area') }, + { id: 'eui', label: $translate.instant('EUI') }, + { id: 'geometry', label: $translate.instant('Geometry') }, + { id: 'ghg', label: $translate.instant('GHG') }, + { id: 'ghg_intensity', label: $translate.instant('GHG Intensity') } + ]; $scope.profiles = [{ id: 0, mappings: [], name: '' }].concat(column_mapping_profiles_payload); $scope.current_profile = $scope.profiles[0] ?? {}; @@ -140,7 +154,8 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ from_field: mapping.name, from_units: mapping.from_units, to_field: mapping.suggestion_column_name || mapping.suggestion || '', - to_table_name: mapping.suggestion_table_name + to_table_name: mapping.suggestion_table_name, + is_omitted: mapping.is_omitted }; const isBuildingSyncProfile = $scope.current_profile.profile_type !== undefined && @@ -292,6 +307,10 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ return col.suggestion_table_name === 'PropertyState' && Boolean(_.find(data_type_columns, { column_name: col.suggestion_column_name })); }; + $scope.is_eui_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'eui'; + $scope.is_area_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'area'; + $scope.is_ghg_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'ghg'; + $scope.is_ghg_intensity_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'ghg_intensity'; const get_default_quantity_units = (col) => { // TODO - hook up to org preferences / last mapping in DB if ($scope.is_data_type_column('eui', col)) { @@ -360,6 +379,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ } if (match) { col.suggestion_column_name = match.column_name; + col.data_type = match.data_type; } else { col.suggestion_column_name = null; } @@ -438,7 +458,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ suggestion: suggestion.to_field, suggestion_column_name: suggestion.to_field, suggestion_table_name: suggestion.to_table_name, - isOmitted: false + is_omitted: suggestion.is_omitted }; }; @@ -464,6 +484,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ } if (match) { col.suggestion = match.display_name; + $scope.change(col, true); } else if ($scope.mappingBuildingSync) { col.suggestion = $filter('titleCase')(col.suggestion_column_name); } @@ -520,14 +541,15 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ $scope.get_mappings = () => { const mappings = []; _.forEach( - $scope.mappings.filter((m) => !m.isOmitted), + $scope.mappings.filter((m) => !m.is_omitted), (col) => { mappings.push({ from_field: col.name, from_units: col.from_units || null, to_field: col.suggestion_column_name || col.suggestion, to_field_display_name: col.suggestion, - to_table_name: col.suggestion_table_name + to_table_name: col.suggestion_table_name, + to_data_type: col.data_type }); } ); @@ -578,7 +600,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ const intersections = _.intersectionWith( required_property_fields, - $scope.mappings.filter((m) => !m.isOmitted), + $scope.mappings.filter((m) => !m.is_omitted), (required_field, raw_col) => _.isMatch(required_field, { column_name: raw_col.suggestion_column_name, inventory_type: raw_col.suggestion_table_name @@ -613,7 +635,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ */ $scope.empty_fields_present = () => Boolean( _.find( - $scope.mappings.filter((m) => !m.isOmitted), + $scope.mappings.filter((m) => !m.is_omitted), { suggestion: '' } ) ); @@ -622,7 +644,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ * empty_units_present: used to disable or enable the 'Map Your Data' button if any units are empty */ $scope.empty_units_present = () => $scope.mappings.some((field) => ( - !field.isOmitted && + !field.is_omitted && field.suggestion_table_name === 'PropertyState' && field.from_units === null && $scope.is_pint_column(field) @@ -633,7 +655,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [ * mappings' button. No warning associated as users "aren't done" listing their mapping settings. */ const suggestions_not_provided_yet = () => { - const non_omitted_mappings = $scope.mappings.filter((m) => !m.isOmitted); + const non_omitted_mappings = $scope.mappings.filter((m) => !m.is_omitted); const no_suggestion_value = Boolean(_.find(non_omitted_mappings, { suggestion: undefined })); const no_suggestion_table_name = Boolean(_.find(non_omitted_mappings, { suggestion_table_name: undefined })); diff --git a/seed/static/seed/partials/column_mappings.html b/seed/static/seed/partials/column_mappings.html index a026598ba8..29fbcc4185 100644 --- a/seed/static/seed/partials/column_mappings.html +++ b/seed/static/seed/partials/column_mappings.html @@ -43,7 +43,7 @@

Column Mappings @@ -110,7 +110,7 @@

Column Mappings - SEED + SEED {$ current_profile.name $} @@ -123,12 +123,13 @@

Column Mappings

Mapped Fields

- + + Is Omitted Inventory Type SEED Header @@ -153,6 +154,9 @@

Column Mappings + + + + - - - + + + - - - @@ -279,10 +289,10 @@

L/m²/year - + {$:: col.name $} - {$:: cell_value $} + {$:: cell_value $} diff --git a/seed/tests/data/mappings/espm-single-mapping.csv b/seed/tests/data/mappings/espm-single-mapping.csv index 101b3df8bb..d2f482ba82 100644 --- a/seed/tests/data/mappings/espm-single-mapping.csv +++ b/seed/tests/data/mappings/espm-single-mapping.csv @@ -1,4 +1,4 @@ -Raw Columns,units,SEED Table,SEED Columns +Raw Columns,units,SEED Table,SEED Columns,isOmitted How Many Buildings?,,PropertyState,building_count City/Municipality,,PropertyState,city Construction Status,,PropertyState,Construction Status diff --git a/seed/tests/data/test_datatype_mapping.csv b/seed/tests/data/test_datatype_mapping.csv new file mode 100644 index 0000000000..5d5fc9e5e2 --- /dev/null +++ b/seed/tests/data/test_datatype_mapping.csv @@ -0,0 +1,14 @@ +Raw Column,Table Name,Field Name,Field Display Name,Field Data Type,Field Unit Type +UBI,TaxLotState,jurisdiction_tax_lot_id,,, +GBA,PropertyState,gross_floor_area,,, +BLDGS,PropertyState,building_count,,, +Address,PropertyState,address_line_1,,, +Owner,PropertyState,owner,,, +City,PropertyState,city,,Integer, +State,PropertyState,state,,, +Zip,PropertyState,postal_code,,, +Property Type,PropertyState,property_type,,, +AYB_YearBuilt,PropertyState,year_built,,, +Custom ID,PropertyState,custom_id_1,,, +PM Property ID,PropertyState,pm_property_id,,, +New Note Field,PropertyState,new_note_field,,string, diff --git a/seed/tests/test_columns.py b/seed/tests/test_columns.py index 84740537e9..d4d75c5e0f 100644 --- a/seed/tests/test_columns.py +++ b/seed/tests/test_columns.py @@ -186,6 +186,16 @@ def test_save_column_mapping_by_file(self): test_mapping, _ = ColumnMapping.get_column_mappings(self.fake_org) self.assertCountEqual(expected, test_mapping) + def test_save_column_mapping_by_file_with_datatypes(self): + self.mapping_import_file = os.path.abspath("./seed/tests/data/test_datatype_mapping.csv") + Column.create_mappings_from_file(self.mapping_import_file, self.fake_org, self.fake_user) + c_note_field = Column.objects.filter(column_name="new_note_field")[0] + self.assertTrue(c_note_field.is_extra_data) + self.assertEqual(c_note_field.data_type, "string") + c_city_field = Column.objects.filter(column_name="city", table_name="PropertyState")[0] + self.assertFalse(c_city_field.is_extra_data) + self.assertEqual(c_city_field.data_type, "string") + def test_column_cant_be_both_extra_data_and_matching_criteria(self): extra_data_column = Column.objects.create( table_name="PropertyState", diff --git a/seed/tests/test_goals.py b/seed/tests/test_goals.py index 4084da86f8..49f8bcfb14 100644 --- a/seed/tests/test_goals.py +++ b/seed/tests/test_goals.py @@ -243,14 +243,14 @@ def reset_goal_data(name): # invalid data goal_data["access_level_instance"] = self.child_ali.id - goal_data["baseline_cycle"] = 999 - goal_data["eui_column1"] = 998 + goal_data["baseline_cycle"] = 9999 + goal_data["eui_column1"] = 9998 response = self.client.post(url, data=json.dumps(goal_data), content_type="application/json") assert response.status_code == 400 errors = response.json() assert errors["name"] == ["goal with this name already exists."] - assert errors["baseline_cycle"] == ['Invalid pk "999" - object does not exist.'] - assert errors["eui_column1"] == ['Invalid pk "998" - object does not exist.'] + assert errors["baseline_cycle"] == ['Invalid pk "9999" - object does not exist.'] + assert errors["eui_column1"] == ['Invalid pk "9998" - object does not exist.'] assert Goal.objects.count() == goal_count # cycles must be unique diff --git a/seed/views/v3/column_mapping_profiles.py b/seed/views/v3/column_mapping_profiles.py index af78492e25..c32ab40a23 100644 --- a/seed/views/v3/column_mapping_profiles.py +++ b/seed/views/v3/column_mapping_profiles.py @@ -90,6 +90,7 @@ def filter(self, request): "from_field": "string", "from_units": "string", "to_table_name": "string", + "omit": "boolean", } ], }, @@ -170,12 +171,12 @@ def csv(self, request, pk=None): return JsonResponse({"status": "error", "data": "No profile with given id"}, status=HTTP_400_BAD_REQUEST) writer = csv.writer(response) - writer.writerow(["Raw Columns", "units", "SEED Table", "SEED Columns"]) + writer.writerow(["Raw Columns", "units", "SEED Table", "SEED Columns", "isOmitted"]) # sort the mappings by the to_field sorted_mappings = sorted(profile.mappings, key=lambda m: m["to_field"].casefold()) for map in sorted_mappings: - writer.writerow([map["from_field"], map["from_units"], map["to_table_name"], map["to_field"]]) + writer.writerow([map["from_field"], map["from_units"], map["to_table_name"], map["to_field"], map.get("is_omitted", False)]) return response @@ -194,6 +195,7 @@ def csv(self, request, pk=None): "from_field": "string", "from_units": "string", "to_table_name": "string", + "is_omitted": "boolean", } ], },