Apply unit conversion to extra data (#4496)

* eeej small files

* option 1

* precommit

* revert

* apply optional units to serialized properties

* precommit

* import unit lookup and apply to mapping

* precommit

* formatting

* taxlots

* precommit

---------

Co-authored-by: Katherine Fleming <[email protected]>

seed/models/tax_lot_properties.py

@@ -17,8 +17,10 @@
 from django.db import models
 from django.db.models import Count
 from django.utils.timezone import make_naive
+from quantityfield.units import ureg
 
 from seed.models.columns import Column
+from seed.serializers.pint import DEFAULT_UNITS
 from seed.utils.geocode import bounding_box_wkt, long_lat_wkt
 from seed.utils.ubid import centroid_wkt
 
@@ -52,23 +54,29 @@ class Meta:
         ]
 
     @classmethod
-    def extra_data_to_dict_with_mapping(cls, instance, mappings, fields=None):
+    def extra_data_to_dict_with_mapping(cls, instance, mappings, fields=None, units={}):
         """
         Convert the extra data to a dictionary with a name mapping for the keys
 
         :param instance: dict, the extra data dictionary
         :param mappings: dict, mapping names { "from_name": "to_name", ...}
         :param fields: list, extra data fields to include. Use the original column names (the ones in the database)
+        :param units: dict, extra data units
+
         :return: dict
         """
         data = {}
 
         if fields:
             for field in fields:
+                value = instance.get(field, None)
+                if units.get(field):
+                    value = value * ureg(units[field])
+
                 if field in mappings:
-                    data[mappings[field]] = instance.get(field, None)
+                    data[mappings[field]] = value
                 else:
-                    data[field] = instance.get(field, None)
+                    data[field] = value
 
         return data
 
@@ -203,8 +211,12 @@ def serialize(
         else:
             filtered_fields = set([col['column_name'] for col in obj_columns if not col['is_extra_data']
                                    and col['id'] in show_columns])
-            filtered_extra_data_fields = set([col['column_name'] for col in obj_columns if col['is_extra_data']
-                                              and col['id'] in show_columns])
+            extra_data_units = {}
+            filtered_extra_data_fields = set()
+            for col in obj_columns:
+                if col['is_extra_data'] and col['id'] in show_columns:
+                    filtered_extra_data_fields.add(col['column_name'])
+                    extra_data_units[col['column_name']] = DEFAULT_UNITS.get(col['data_type'])
 
         # get the related data
         join_map = {}
@@ -228,7 +240,8 @@ def serialize(
                     TaxLotProperty.extra_data_to_dict_with_mapping(
                         obj.state.extra_data,
                         obj_column_name_mapping,
-                        fields=filtered_extra_data_fields
+                        fields=filtered_extra_data_fields,
+                        units=extra_data_units
                     ).items()
                 )
 

seed/serializers/pint.py

@@ -34,6 +34,13 @@
 GHG_DEFAULT_UNITS = 'MtCO2e/year'
 GHG_INTENSITY_DEFAULT_UNITS = 'kgCO2e/ft**2/year'
 
+DEFAULT_UNITS = {
+    'area': AREA_DEFAULT_UNITS,
+    'eui': EUI_DEFAULT_UNITS,
+    'ghg': GHG_DEFAULT_UNITS,
+    'ghg_intensity': GHG_INTENSITY_DEFAULT_UNITS,
+}
+
 
 def to_raw_magnitude(obj):
     return "{:.2f}".format(obj.magnitude)

seed/views/v3/import_files.py

@@ -54,7 +54,7 @@
     get_column_mapping,
     obj_to_dict
 )
-from seed.serializers.pint import apply_display_unit_preferences
+from seed.serializers.pint import DEFAULT_UNITS, apply_display_unit_preferences
 from seed.utils.api import OrgMixin, api_endpoint_class
 from seed.utils.api_schema import (
     AutoSchemaHelper,
@@ -363,6 +363,7 @@ def mapping_results(self, request, pk=None):
         columns_from_db = Column.retrieve_all(org_id)
         property_column_name_mapping = {}
         taxlot_column_name_mapping = {}
+        extra_data_units = {}
         for field_name in field_names:
             # find the field from the columns in the database
             for column in columns_from_db:
@@ -372,14 +373,16 @@ def mapping_results(self, request, pk=None):
                     property_column_name_mapping[column['column_name']] = column['name']
                     if not column['is_extra_data']:
                         fields['PropertyState'].append(field_name[1])  # save to the raw db fields
-                    continue
+                    elif DEFAULT_UNITS.get(column['data_type']):
+                        extra_data_units[column['column_name']] = DEFAULT_UNITS.get(column['data_type'])
                 elif column['table_name'] == 'TaxLotState' and \
                         field_name[0] == 'TaxLotState' and \
                         field_name[1] == column['column_name']:
                     taxlot_column_name_mapping[column['column_name']] = column['name']
                     if not column['is_extra_data']:
                         fields['TaxLotState'].append(field_name[1])  # save to the raw db fields
-                    continue
+                    elif DEFAULT_UNITS.get(column['data_type']):
+                        extra_data_units[column['column_name']] = DEFAULT_UNITS.get(column['data_type'])
 
         inventory_type = request.data.get('inventory_type', 'all')
 
@@ -408,6 +411,7 @@ def mapping_results(self, request, pk=None):
                         prop.extra_data,
                         property_column_name_mapping,
                         fields=prop.extra_data.keys(),
+                        units=extra_data_units
                     ).items()
                 )