Skip to content

Commit

Permalink
Enable data type setting during data import (#4740)
Browse files Browse the repository at this point in the history 
* include data type mapping column in front end, set the data type for extra data columns only

* only look at data_type for extra_data columns

* enable additional columns in csv for to_display_name, to_data_type and to_unit_type

* ensure files with the additional columns work properly for extra data fields, and do not overwrite canonical column data types

* add file for datatype mapping test

* add isOmitted to mapping, respect omitted columns when flagged as such in a mapping profile

* default to false for missing omit column

* linting, remove log call

* lint fix

* one more linter fix

* ensure types line up

* Add messages for reasons the save button on column mappings is disabled (#4744)

* add text to save button tooltip

* simplify error message handling for the profile save button tooltip

* include ghg and ghg_intensity columns in the danger class check

* isOmitted -> is_omitted in mappings for consistency

* Increase invalid goal ids

---------

Co-authored-by: Katherine Fleming <[email protected]>
Co-authored-by: Alex Swindler <[email protected]>
  • Loading branch information
@crutan @kflemin @axelstudios
3 people authored Sep 24, 2024
1 parent 8bee67e commit d9e521f
Show file tree
Hide file tree
Showing 11 changed files with 134 additions and 56 deletions.
4 changes: 4 additions & 0 deletions seed/models/column_mapping_profiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ def create_from_file(
"to_table_name": row[2],
"to_field": row[3],
}
try:
data["is_omitted"] = "True" if row[4].lower().strip() == "true" else "False"
except IndexError:
data["is_omitted"] = "False"
mappings.append(data)
else:
raise Exception(f"Mapping file does not exist: {filename}")
Expand Down
50 changes: 26 additions & 24 deletions seed/models/columns.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,23 @@ class Column(models.Model):
"PointField": "geometry",
}

DB_TYPES = {
"number": "float",
"float": "float",
"integer": "integer",
"string": "string",
"geometry": "geometry",
"datetime": "datetime",
"date": "date",
"boolean": "boolean",
"area": "float",
"eui": "float",
"ghg": "float",
"ghg_intensity": "float",
"wui": "float",
"water_use": "float",
}

DATA_TYPE_PARSERS: dict[str, Callable] = {
"number": lambda v: float(v.replace(",", "") if isinstance(v, str) else v),
"float": lambda v: float(v.replace(",", "") if isinstance(v, str) else v),
Expand Down Expand Up @@ -1010,9 +1027,9 @@ def create_mappings_from_file(filename, organization, user, import_file_id=None)
"from_field": row[0],
"to_table_name": row[1],
"to_field": row[2],
# "to_display_name": row[3],
# "to_data_type": row[4],
# "to_unit_type": row[5],
"to_display_name": row[3],
"to_data_type": row[4],
"to_unit_type": row[5],
}
mappings.append(data)
else:
Expand Down Expand Up @@ -1059,13 +1076,13 @@ def create_mappings(mappings, organization, user, import_file_id=None):
'to_field': 'gross_floor_area',
'to_field_display_name': 'Gross Floor Area',
'to_table_name': 'PropertyState',
'to_data_type': 'string', # an internal data type mapping
}
]
"""

# initialize a cache to store the mappings
cache_column_mapping = []

# Take the existing object and return the same object with the db column objects added to
# the dictionary (to_column_object and from_column_object)
mappings = Column._column_fields_to_columns(mappings, organization, user)
Expand Down Expand Up @@ -1101,7 +1118,6 @@ def create_mappings(mappings, organization, user, import_file_id=None):

column_mapping.user = user
column_mapping.save()

cache_column_mapping.append(
{
"from_field": mapping["from_field"],
Expand Down Expand Up @@ -1176,6 +1192,10 @@ def _column_fields_to_columns(fields, organization, user):
"table_name": "" if is_ah_data else field["to_table_name"],
"is_extra_data": is_extra_data,
}
# Only compare against data type if it is provided && the column is an extra data column
if ("to_data_type" in field) and (is_extra_data):
to_col_params["data_type"] = field["to_data_type"]

if is_root_user or is_ah_data:
to_org_col, _ = Column.objects.get_or_create(**to_col_params)
else:
Expand Down Expand Up @@ -1312,28 +1332,10 @@ def retrieve_db_types():
"""
columns = copy.deepcopy(Column.DATABASE_COLUMNS)

# TODO: There seem to be lots of these lists floating around. We should consolidate them.
MAP_TYPES = {
"number": "float",
"float": "float",
"integer": "integer",
"string": "string",
"geometry": "geometry",
"datetime": "datetime",
"date": "date",
"boolean": "boolean",
"area": "float",
"eui": "float",
"ghg": "float",
"ghg_intensity": "float",
"wui": "float",
"water_use": "float",
}

types = OrderedDict()
for c in columns:
try:
types[c["column_name"]] = MAP_TYPES[c["data_type"]]
types[c["column_name"]] = Column.DB_TYPES[c["data_type"]]
except KeyError:
_log.error("could not find data_type for %s" % c)
types[c["column_name"]] = ""
Expand Down
10 changes: 10 additions & 0 deletions seed/static/seed/js/controllers/column_mappings_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,11 +423,21 @@ angular.module('SEED.controller.column_mappings', []).controller('column_mapping
return Boolean(_.find(_.values(grouped_by_from_field), (group) => group.length > 1));
};

$scope.save_button_tooltip = () => {
if (!$scope.changes_possible) return 'No changes';
if ($scope.header_duplicates_present()) return 'Duplicate headers present';
if ($scope.empty_units_present()) return 'Empty units present';
if ($scope.current_profile.profile_type === COLUMN_MAPPING_PROFILE_TYPE_BUILDINGSYNC_DEFAULT) return 'This profile cannot be changed';
if (!$scope.profile_action_ok('update')) return 'Unknown profile type';
return 'Save';
};

$scope.empty_units_present = () => Boolean(
_.find($scope.current_profile.mappings, (field) => {
const has_units = $scope.is_pint_column(field);
return field.to_table_name === 'PropertyState' && field.from_units === null && has_units;
})

);

$scope.profile_action_ok = (action) => {
Expand Down
38 changes: 30 additions & 8 deletions seed/static/seed/js/controllers/mapping_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,20 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
COLUMN_MAPPING_PROFILE_TYPE_BUILDINGSYNC_CUSTOM,
derived_columns_payload
) {
$scope.data_types = [
{ id: 'None', label: 'None' },
{ id: 'number', label: $translate.instant('Number') },
{ id: 'integer', label: $translate.instant('Integer') },
{ id: 'string', label: $translate.instant('Text') },
{ id: 'datetime', label: $translate.instant('Datetime') },
{ id: 'date', label: $translate.instant('Date') },
{ id: 'boolean', label: $translate.instant('Boolean') },
{ id: 'area', label: $translate.instant('Area') },
{ id: 'eui', label: $translate.instant('EUI') },
{ id: 'geometry', label: $translate.instant('Geometry') },
{ id: 'ghg', label: $translate.instant('GHG') },
{ id: 'ghg_intensity', label: $translate.instant('GHG Intensity') }
];
$scope.profiles = [{ id: 0, mappings: [], name: '<None selected>' }].concat(column_mapping_profiles_payload);

$scope.current_profile = $scope.profiles[0] ?? {};
Expand Down Expand Up @@ -140,7 +154,8 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
from_field: mapping.name,
from_units: mapping.from_units,
to_field: mapping.suggestion_column_name || mapping.suggestion || '',
to_table_name: mapping.suggestion_table_name
to_table_name: mapping.suggestion_table_name,
is_omitted: mapping.is_omitted
};
const isBuildingSyncProfile =
$scope.current_profile.profile_type !== undefined &&
Expand Down Expand Up @@ -292,6 +307,10 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
return col.suggestion_table_name === 'PropertyState' && Boolean(_.find(data_type_columns, { column_name: col.suggestion_column_name }));
};

$scope.is_eui_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'eui';
$scope.is_area_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'area';
$scope.is_ghg_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'ghg';
$scope.is_ghg_intensity_column = (col) => col.suggestion_table_name === 'PropertyState' && col.data_type === 'ghg_intensity';
const get_default_quantity_units = (col) => {
// TODO - hook up to org preferences / last mapping in DB
if ($scope.is_data_type_column('eui', col)) {
Expand Down Expand Up @@ -360,6 +379,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
}
if (match) {
col.suggestion_column_name = match.column_name;
col.data_type = match.data_type;
} else {
col.suggestion_column_name = null;
}
Expand Down Expand Up @@ -438,7 +458,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
suggestion: suggestion.to_field,
suggestion_column_name: suggestion.to_field,
suggestion_table_name: suggestion.to_table_name,
isOmitted: false
is_omitted: suggestion.is_omitted
};
};

Expand All @@ -464,6 +484,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
}
if (match) {
col.suggestion = match.display_name;
$scope.change(col, true);
} else if ($scope.mappingBuildingSync) {
col.suggestion = $filter('titleCase')(col.suggestion_column_name);
}
Expand Down Expand Up @@ -520,14 +541,15 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
$scope.get_mappings = () => {
const mappings = [];
_.forEach(
$scope.mappings.filter((m) => !m.isOmitted),
$scope.mappings.filter((m) => !m.is_omitted),
(col) => {
mappings.push({
from_field: col.name,
from_units: col.from_units || null,
to_field: col.suggestion_column_name || col.suggestion,
to_field_display_name: col.suggestion,
to_table_name: col.suggestion_table_name
to_table_name: col.suggestion_table_name,
to_data_type: col.data_type
});
}
);
Expand Down Expand Up @@ -578,7 +600,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [

const intersections = _.intersectionWith(
required_property_fields,
$scope.mappings.filter((m) => !m.isOmitted),
$scope.mappings.filter((m) => !m.is_omitted),
(required_field, raw_col) => _.isMatch(required_field, {
column_name: raw_col.suggestion_column_name,
inventory_type: raw_col.suggestion_table_name
Expand Down Expand Up @@ -613,7 +635,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
*/
$scope.empty_fields_present = () => Boolean(
_.find(
$scope.mappings.filter((m) => !m.isOmitted),
$scope.mappings.filter((m) => !m.is_omitted),
{ suggestion: '' }
)
);
Expand All @@ -622,7 +644,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
* empty_units_present: used to disable or enable the 'Map Your Data' button if any units are empty
*/
$scope.empty_units_present = () => $scope.mappings.some((field) => (
!field.isOmitted &&
!field.is_omitted &&
field.suggestion_table_name === 'PropertyState' &&
field.from_units === null &&
$scope.is_pint_column(field)
Expand All @@ -633,7 +655,7 @@ angular.module('SEED.controller.mapping', []).controller('mapping_controller', [
* mappings' button. No warning associated as users "aren't done" listing their mapping settings.
*/
const suggestions_not_provided_yet = () => {
const non_omitted_mappings = $scope.mappings.filter((m) => !m.isOmitted);
const non_omitted_mappings = $scope.mappings.filter((m) => !m.is_omitted);
const no_suggestion_value = Boolean(_.find(non_omitted_mappings, { suggestion: undefined }));
const no_suggestion_table_name = Boolean(_.find(non_omitted_mappings, { suggestion_table_name: undefined }));

Expand Down
10 changes: 7 additions & 3 deletions seed/static/seed/partials/column_mappings.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ <h2><i class="fa-solid fa-sitemap"></i> <span translate>Column Mappings</span></
ng-click="save_profile()"
ng-disabled="!changes_possible || header_duplicates_present() || empty_units_present() || !profile_action_ok('update')"
tooltip-placement="bottom"
uib-tooltip="Save"
uib-tooltip="{$ save_button_tooltip() $}"
>
<i class="fa-solid fa-check"></i>
</button>
Expand Down Expand Up @@ -110,7 +110,7 @@ <h2><i class="fa-solid fa-sitemap"></i> <span translate>Column Mappings</span></
<table class="table table-striped">
<thead>
<tr>
<th colspan="3" class="source_data">SEED</th>
<th colspan="4" class="source_data">SEED</th>
<th colspan="6" class="source_data">{$ current_profile.name $}</th>
</tr>
</thead>
Expand All @@ -123,12 +123,13 @@ <h2><i class="fa-solid fa-sitemap"></i> <span translate>Column Mappings</span></
</span>
</td>
<td style="border-right: 0 none"><h3 translate>Mapped Fields</h3></td>
<td style="max-width: 128px; width: 128px"></td>
<td colspan="2" style="max-width: 128px; width: 128px"></td>
<td colspan="6"></td>
</tr>
</tbody>
<thead>
<tr>
<th><translate>Is Omitted</translate></th>
<th><translate>Inventory Type</translate></th>
<th class="mapping_field ellipsis-resizable" ng-click="sort_by('seed')" sd-resizable>
<span><translate>SEED Header</translate><i class="fa-solid {$ get_sort_icon('seed') $} pad-left-5"></i></span>
Expand All @@ -153,6 +154,9 @@ <h2><i class="fa-solid fa-sitemap"></i> <span translate>Column Mappings</span></
</thead>
<tbody id="mapped-table">
<tr ng-repeat="(index, col) in dropdown_selected_profile.mappings">
<td style="text-align: center; width: min-content" ng-attr-id="mapped-row-is_omitted-{$:: $index $}">
<input type="checkbox" ng-model="col.is_omitted" ng-disabled="import_file.matching_done" ng-change="flag_change(col)" />
</td>
<td style="text-align: right" ng-attr-id="mapped-row-type-{$:: $index $}">
<select ng-model="col.to_table_name" ng-change="updateSingleInventoryTypeDropdown()">
<option value="PropertyState" translate>Property</option>
Expand Down
Loading

0 comments on commit d9e521f

Please sign in to comment.