diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbd7acc..73157fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: check-added-large-files @@ -35,12 +35,12 @@ repos: "--ignore=E501,E402,W503,W504,E731", ] - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 args: ["--ignore=E501,E402,W503,W504,E731,F401"] - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v4.0.0-alpha.8 hooks: - id: prettier types_or: [css, yaml, markdown, html, scss, javascript] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cd0c5b8..91f6199 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,19 @@ Changelog ========= +0.5.0 +----- + +## What's Changed + +* Add PyPi release action by @nllong in https://github.com/SEED-platform/py-seed/pull/42 +* Update precommit versions by @nllong in https://github.com/SEED-platform/py-seed/pull/44 +* Add create organization and retrieve property cross cycle data by @nllong in https://github.com/SEED-platform/py-seed/pull/45 +* Add is_omitted column to column mapping profiles by @crutan in https://github.com/SEED-platform/py-seed/pull/46 + +**Full Changelog**: https://github.com/SEED-platform/py-seed/compare/v0.4.3...v0.5.0 + + 0.4.3 ----- diff --git a/README.rst b/README.rst index 9fd3e62..d96ed71 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,20 @@ More information can be found here: * https://github.com/SEED-platform/pyseed-examples +Compatibility Matrix +------------- + +.. list-table:: + :widths: 50 50 + :header-rows: 1 + + * - py-SEED Version + - SEED Version + * - 0.5.0 + - 3.1.0 + * - 0.4.3 + - 2.21.0 - 3.0.0 + Stakeholders ------------- @@ -133,5 +147,6 @@ This project is configured with GitHub Actions to automatically release to PyPi * Once deployed to main, create a new tag in GitHub against main and copy the change log notes into the tag description * GitHub Actions will automatically prepare the release the new version to PyPi * Go to GitHub actions to approve the release +* After merging into main, then in the command line with the develop branch run `git merge origin main` and push the changes. This might have to be done with a person with elevated privileges to bypass the protected branch settings. The GitHub Action required updates to the GitHub repo to only release on tags (https://github.com/SEED-platform/py-seed/settings/environments) after approval and on PyPi to add an authorized publisher (https://pypi.org/manage/project/py-SEED/settings/publishing/). diff --git a/pyseed/seed_client.py b/pyseed/seed_client.py index a26bfe5..6b7b0e4 100644 --- a/pyseed/seed_client.py +++ b/pyseed/seed_client.py @@ -153,6 +153,18 @@ def instance_information(self) -> dict: info["username"] = self.client.username return info + def get_users(self) -> dict: + """Get a list of users visible to the current user + + Returns: + dict: { "users": [{ + "email": "abc@def.com", + "user_id": 1 + }]} + """ + users = self.client.list(endpoint="users") + return users + def get_organizations(self, brief: bool = True) -> dict: """Get a list organizations (that one is allowed to view) @@ -179,6 +191,67 @@ def get_organizations(self, brief: bool = True) -> dict: ) return orgs + def get_user_id(self, username: str) -> Union[None, int]: + """Get the user ID for the given username + + Args: + username (str): username to get the ID for + + Returns: + int: user ID + """ + for user in self.get_users()['users']: + # compare string case insensitive + if user["email"].lower() == username.lower(): + return user["user_id"] + + return None + + def create_organization(self, org_name: str) -> dict: + """Create an organization with the given name + + Args: + org_name (str): name of the organization to create + + Returns: + dict: { + 'status': 'success', + 'message': 'Organization created', + 'organization': { + 'name': 'NEW ORG', + 'org_id': 17, + 'id': 17, + 'number_of_users': 1, + 'user_is_owner': True, + 'user_role': 'owner', + 'owners': [...], + 'sub_orgs': [...], + 'is_parent': True, + 'parent_id': 17, + ... + 'display_units_eui': 'kBtu/ft**2/year', + 'cycles': [...], + 'created': '2024-06-13', + 'mapquest_api_key': '', + + } + } + """ + # see if the organization already exists + orgs = self.get_organizations() + for org in orgs: + if org["name"].lower() == org_name.lower(): + raise Exception(f"Organization '{org_name}' already exists") + + user_id = self.get_user_id(self.client.username) + + payload = { + "user_id": user_id, + "organization_name": org_name, + } + org = self.client.post(endpoint="organizations", json=payload) + return org + def get_buildings(self) -> list[dict]: total_qry = self.client.list(endpoint="properties", data_name="pagination", per_page=100) @@ -584,13 +657,14 @@ def get_or_create_cycle( end_date: date, set_cycle_id: bool = False, ) -> dict: - """Get or create a new cycle. If the cycle_name already exists, then it simply returns the existing cycle. However, if the cycle_name does not exist, then it will create a new cycle. + """Get or create a new cycle. If the cycle_name already exists, then it simply returns + the existing cycle. However, if the cycle_name does not exist, then it will create a new cycle. Args: cycle_name (str): name of the cycle to get or create start_date (date): MM/DD/YYYY of start date cycle end_date (date): MM/DD/YYYY of end data for cycle - set_cycle_id (str): Set the object's cycle_id to the resulting cycle that is returned (either existing or newly created) + set_cycle_id (str): Set the object's cycle_id to the resulting cycle that is returned (either existing or newly created) Returns: dict: { @@ -850,6 +924,7 @@ def create_or_update_column_mapping_profile( "from_units": null, "to_table_name": "PropertyState" "to_field": "address_line_1", + "is_omitted": False }, { "from_field": "address1", @@ -860,6 +935,7 @@ def create_or_update_column_mapping_profile( ... ] + The is_omitted mapping may be absent - it is treated as False if it is not present. Returns: dict: { 'id': 1 @@ -898,9 +974,9 @@ def create_or_update_column_mapping_profile_from_file( ) -> dict: """creates or updates a mapping profile. The format of the mapping file is a CSV with the following format: - Raw Columns, units, SEED Table, SEED Columns\n - PM Property ID, , PropertyState, pm_property_id\n - Building ID, , PropertyState, custom_id_1\n + Raw Columns, units, SEED Table, SEED Columns, Omit\n + PM Property ID, , PropertyState, pm_property_id, False\n + Building ID, , PropertyState, custom_id_1, False\n ...\n This only works for 'Normal' column mapping profiles, that is, it does not work for @@ -949,7 +1025,7 @@ def set_import_file_column_mappings( ) def get_columns(self) -> dict: - """get the list of columns. + """Get the list of columns Returns: dict: { @@ -961,7 +1037,8 @@ def get_columns(self) -> dict: return result def create_extra_data_column(self, column_name: str, display_name: str, inventory_type: str, column_description: str, data_type: str) -> dict: - """ create an extra data column. If column exists, skip + """Create an extra data column. If column exists, skip + Args: 'column_name': 'project_type', 'display_name': 'Project Type', @@ -979,7 +1056,6 @@ def create_extra_data_column(self, column_name: str, display_name: str, inventor } } """ - # get extra data columns (only) result = self.client.list(endpoint="columns") columns = result['columns'] @@ -1005,7 +1081,8 @@ def create_extra_data_column(self, column_name: str, display_name: str, inventor return result def create_extra_data_columns_from_file(self, columns_csv_filepath: str) -> list: - """ create extra data columns from a csv file. if column exist, skip. + """Create extra data columns from a csv file. if column exist, skip. + Args: 'columns_csv_filepath': 'path/to/file' file is expected to have headers: column_name, display_name, column_description, @@ -1156,11 +1233,13 @@ def get_meter_data(self, property_id, interval: str = 'Exact', excluded_meter_id def start_save_data(self, import_file_id: int, multiple_cycle_upload: bool = False) -> dict: """start the background process to save the data file to the database. - This is the state before the mapping. + This is the state before the mapping. If multiple_cycle_upload is set to True, then the + importing file's year_ending column will be used to determine the cycle. Note that the + cycles must be created in SEED for the multiple cycle upload to work correctly Args: import_file_id (int): id of the import file to save - multiple_cycle_upload (bool): whether to use multiple cycle upload + multiple_cycle_upload (bool): whether to use multiple cycle upload. Returns: dict: progress key @@ -1435,8 +1514,12 @@ def upload_and_match_datafile( column_mapping_profile_name (str): Name of the column mapping profile to use column_mappings_file (str): Mapping that will be uploaded to the column_mapping_profile_name import_meters_if_exist (bool): If true, will import meters from the meter tab if they exist in the datafile. Defaults to False. + + Kwargs: + datafile_type (str): Type of datafile multiple_cycle_upload (bool): Whether to use multiple cycle upload. Defaults to False. + Returns: dict: { matching summary @@ -1707,3 +1790,22 @@ def retrieve_analysis_result(self, analysis_id: int, analysis_view_id: int) -> d url_args={"PK": analysis_id, "ANALYSIS_VIEW_PK": analysis_view_id}, include_org_id_query_param=True, ) + + def get_cross_cycle_data(self, property_view_id: int) -> dict: + """Retrieve the cross cycle data for a property. This is the data that + is shared across all the cycles used to populate a property's cross + cycle view. + + Args: + property_view_id (int): Property view id + + Returns: + dict: Cross cycle data for the property view + """ + return self.client.get( + None, + required_pk=False, + endpoint="properties_cross_cycle_data", + url_args={"PK": property_view_id}, + include_org_id_query_param=True, + ) diff --git a/pyseed/seed_client_base.py b/pyseed/seed_client_base.py index 51bcbfa..107f28d 100644 --- a/pyseed/seed_client_base.py +++ b/pyseed/seed_client_base.py @@ -75,6 +75,7 @@ 'audit_template_building_xml': '/api/v3/audit_template/PK/get_building_xml', 'audit_template_submission': '/api/v3/audit_template/PK/get_submission', 'import_files_matching_results': '/api/v3/import_files/PK/matching_and_geocoding_results/', + 'properties_cross_cycle_data': '/api/v3/properties/PK/links/', 'progress': '/api/v3/progress/PROGRESS_KEY/', 'properties_analyses': '/api/v3/properties/PK/analyses/', 'properties_meter_usage': '/api/v3/properties/PK/meter_usage/', @@ -250,7 +251,7 @@ def _check_response(self, response, *args, **kwargs): # this is a system matching response, which is okay. return the success flag of this status_flag = response.json()['progress_data'].get('status', None) error = status_flag not in ['not-started', 'success', 'parsing'] - elif not any(key in ['results', 'readings', 'data', 'status', 'id', 'organizations', 'sha'] for key in response.json().keys()): + elif not any(key in ['results', 'readings', 'data', 'status', 'id', 'organizations', 'sha', 'users'] for key in response.json().keys()): # In some cases there is not a 'status' field, so check if there are # any other keys in the response that depict a success: # readings - this comes from meters diff --git a/pyseed/utils.py b/pyseed/utils.py index e10fd59..ee09ef4 100644 --- a/pyseed/utils.py +++ b/pyseed/utils.py @@ -113,15 +113,18 @@ def read_map_file(mapfile_path): # Open the mapping file and fill list maplist = list() - for rowitem in map_reader: - maplist.append( - { - 'from_field': rowitem[0], - 'from_units': rowitem[1], - 'to_table_name': rowitem[2], - 'to_field': rowitem[3], - } - ) + data = { + "from_field": rowitem[0], + "from_units": rowitem[1], + "to_table_name": rowitem[2], + "to_field": rowitem[3], + } + try: + data["is_omitted"] = True if rowitem[4].lower().strip() == "true" else False + except IndexError: + data["is_omitted"] = False + + maplist.append(data) return maplist diff --git a/setup.cfg b/setup.cfg index 5fee117..c77b0b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name=py-seed -version=0.4.3 +version=0.5.0 description=A Python API client for the SEED Platform author=Nicholas Long, Katherine Fleming, Fable Turas, Paul Munday author_email=nicholas.long@nrel.gov, fable@raintechpdx.com, paul@paulmunday.net diff --git a/tests/data/test-seed-data-mappings.csv b/tests/data/test-seed-data-mappings.csv index 9e17232..d32fd0e 100644 --- a/tests/data/test-seed-data-mappings.csv +++ b/tests/data/test-seed-data-mappings.csv @@ -8,7 +8,7 @@ Sq. Ft,ft**2,PropertyState,gross_floor_area Total GHG Emissions Intensity,kgCO2e/ft**2/year,PropertyState,total_ghg_emissions_intensity Site EUI,kBtu/ft**2/year,PropertyState,site_eui PM Release Date,,PropertyState,release_date -Year Ending,,PropertyState,Year Ending Excel +Year Ending,,PropertyState,year_ending GHGI Target,,PropertyState,GHGI Target GHGI Target Year,,PropertyState,GHGI Target Year EUI Target,,PropertyState,EUI Target diff --git a/tests/data/test-seed-data-with-multiple-cycles.xlsx b/tests/data/test-seed-data-with-multiple-cycles.xlsx new file mode 100755 index 0000000..b84c2ea Binary files /dev/null and b/tests/data/test-seed-data-with-multiple-cycles.xlsx differ diff --git a/tests/test_seed_client.py b/tests/test_seed_client.py index baeb588..4a31f78 100644 --- a/tests/test_seed_client.py +++ b/tests/test_seed_client.py @@ -13,6 +13,9 @@ # Local Imports from pyseed.seed_client import SeedClient +# For CI the test org is 1, but for local testing it may be different +ORGANIZATION_ID = 1 + @pytest.mark.integration class SeedClientTest(unittest.TestCase): @@ -23,7 +26,7 @@ def setup_class(cls): if not cls.output_dir.exists(): cls.output_dir.mkdir() - cls.organization_id = 1 + cls.organization_id = ORGANIZATION_ID # The seed-config.json file needs to be added to the project root directory # If running SEED locally for testing, then you can run the following from your SEED root directory: @@ -59,6 +62,17 @@ def test_seed_client_info(self): info = self.seed_client.instance_information() assert set(("version", "sha")).issubset(info.keys()) + def test_create_organization(self): + # create a new organization. This test requires that the + # org does not already exist, which is common in the CI. + org = self.seed_client.create_organization("NEW ORG") + assert org["organization"]["id"] is not None + + # try to create again and it should raise an error + with self.assertRaises(Exception) as excpt: + self.seed_client.create_organization("NEW ORG") + assert "already exists" in str(excpt.exception) + def test_seed_buildings(self): # set cycle before retrieving (just in case) self.seed_client.get_cycle_by_name('pyseed-api-test', set_cycle_id=True) @@ -400,3 +414,71 @@ def test_upload_espm_property_to_seed(self): # response = self.seed_client.retrieve_at_building_and_update(self, at_building_id, self.cycle_id, building['id']) # self.assertTrue(response['status'] == 'success') + + +@pytest.mark.integration +class SeedClientMultiCycleTest(unittest.TestCase): + @classmethod + def setup_class(cls): + """setup for all of the tests below""" + cls.output_dir = Path("tests/output") + if not cls.output_dir.exists(): + cls.output_dir.mkdir() + + # Use the default organization to create the client, + # but this will be overwritten in the test class below. + cls.organization_id = ORGANIZATION_ID + + # The seed-config.json file needs to be added to the project root directory + # If running SEED locally for testing, then you can run the following from your SEED root directory: + # ./manage.py create_test_user_json --username user@seed-platform.org --file ../py-seed/seed-config.json --pyseed + config_file = Path("seed-config.json") + cls.seed_client = SeedClient( + cls.organization_id, connection_config_filepath=config_file + ) + + @classmethod + def teardown_class(cls): + # remove all of the test buildings? + pass + + def test_upload_multiple_cycles_and_read_back(self): + # Get/create the new cycle and upload the data. Make sure to set the cycle ID so that the + # data end up in the correct cycle + new_org = self.seed_client.create_organization("pyseed-multi-cycle") + self.seed_client.client.org_id = new_org["organization"]["id"] + + for year_start, year_end in [(2020, 2021), (2021, 2022), (2022, 2023)]: + self.seed_client.get_or_create_cycle( + f"pyseed-multi-cycle-test-{year_start}", + date(year_start, 6, 1), + date(year_end, 6, 1), + set_cycle_id=True, + ) + + # due to structure of loop above, the cycle_id is set to the last cycle + + result = self.seed_client.upload_and_match_datafile( + "pyseed-multiple-cycle-test", + "tests/data/test-seed-data-with-multiple-cycles.xlsx", + "Single Step Column Mappings", + "tests/data/test-seed-data-mappings.csv", + import_meters_if_exist=False, + multiple_cycle_upload=True + ) + + assert result is not None + + # retrieve the single building + building = self.seed_client.search_buildings(identifier_exact=11111) + assert len(building) == 1 + property_view_id = building[0]["id"] + + # retrieve cross cycle + building_cycles = self.seed_client.get_cross_cycle_data(property_view_id) + + assert len(building_cycles) == 3 + # check that the site_euis are correct + assert building_cycles[0]['site_eui'] == 95 + assert building_cycles[1]['site_eui'] == 181 + assert building_cycles[2]['site_eui'] == 129 diff --git a/tests/test_utils.py b/tests/test_utils.py index e7d1cc4..7e1fd71 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -22,5 +22,6 @@ def test_mapping_file(self): "from_units": "ft**2", "to_field": "gross_floor_area", "to_table_name": "PropertyState", + "is_omitted": False } assert mappings[5] == expected