diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd4af5..7e2ee11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,9 @@ jobs: test_env: [python, precommit, mypy] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Display system info @@ -36,9 +36,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.9" - name: Install dependencies @@ -66,7 +66,7 @@ jobs: - name: Wait for web server uses: nev7n/wait_for_response@v1 with: - # Increase the timeout significanlty, the EEEJ census tract + # Increase the timeout significantly, the EEEJ census tract # migration take ~6 minutes to run along. url: "http://localhost:8000/" responseCode: 200 @@ -86,7 +86,7 @@ jobs: SEED_PM_UN: ${{ secrets.SEED_PM_UN }} SEED_PM_PW: ${{ secrets.SEED_PM_PW }} run: | - pytest -m "integration" -s + pytest -m integration -s - name: Dump docker logs on failure if: failure() uses: jwalton/gh-docker-logs@v2 diff --git a/cspell.json b/cspell.json index 29e58b8..98107d9 100644 --- a/cspell.json +++ b/cspell.json @@ -11,14 +11,20 @@ "dname", "durl", "ECAM", + "EEEJ", "ESPM", "excpt", "geocoded", "greenbuildingregistry", + "jakejarvis", "JSONAPI", + "jwalton", "Munday", + "ndeloof", "officedocument", "openxmlformats", + "precommit", + "printenv", "pyseed", "pytest", "sdist", diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..976ba02 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/pyseed/seed_client.py b/pyseed/seed_client.py index b7d175b..a26bfe5 100644 --- a/pyseed/seed_client.py +++ b/pyseed/seed_client.py @@ -4,17 +4,18 @@ """ # Imports from Standard Library -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union # Imports from Third Party Modules import json import logging +import os import time from collections import Counter from csv import DictReader from datetime import date +from openpyxl import Workbook from pathlib import Path -from urllib.parse import _NetlocResultMixinStr # Local Imports from pyseed.seed_client_base import SEEDReadWriteClient @@ -104,8 +105,8 @@ class SeedClient(SeedClientWrapper): def __init__( self, organization_id: int, - connection_params: dict = None, - connection_config_filepath: Path = None, + connection_params: Optional[dict] = None, + connection_config_filepath: Optional[Path] = None, ) -> None: super().__init__(organization_id, connection_params, connection_config_filepath) @@ -152,13 +153,13 @@ def instance_information(self) -> dict: info["username"] = self.client.username return info - def get_organizations(self, brief: bool = True) -> Dict: + def get_organizations(self, brief: bool = True) -> dict: """Get a list organizations (that one is allowed to view) Args: brief (bool, optional): if True, then only return the organization id with some other basic info. Defaults to True. Returns: - Dict: [ + dict: [ { "name": "test-org", "org_id": 1, @@ -178,11 +179,11 @@ def get_organizations(self, brief: bool = True) -> Dict: ) return orgs - def get_buildings(self) -> List[dict]: + def get_buildings(self) -> list[dict]: total_qry = self.client.list(endpoint="properties", data_name="pagination", per_page=100) # step through each page of the results - buildings: List[dict] = [] + buildings: list[dict] = [] for i in range(1, total_qry['num_pages'] + 1): buildings = buildings + self.client.list( endpoint="properties", @@ -220,7 +221,7 @@ def get_property(self, property_view_id: int) -> dict: """Return a single property by the property view id. Args: - property__id (int): ID of the property to return. This is the ID that is in the URL http://SEED_URL/app/#/properties/{property_view_id} + property_view_id (int): ID of the property view with a property to return. This is the ID that is in the URL http://SEED_URL/app/#/properties/{property_view_id} Returns: dict: { @@ -240,11 +241,14 @@ def get_property(self, property_view_id: int) -> dict: ) def search_buildings( - self, identifier_filter: str = None, identifier_exact: str = None, cycle_id: int = None + self, + identifier_filter: Optional[str] = None, + identifier_exact: Optional[str] = None, + cycle_id: Optional[int] = None, ) -> dict: if not cycle_id: cycle_id = self.cycle_id - payload: Dict[str, Any] = { + payload: dict[str, Any] = { "cycle": cycle_id, } if identifier_filter is not None: @@ -258,11 +262,11 @@ def search_buildings( ) return properties - def get_labels(self, filter_by_name: list = None) -> list: + def get_labels(self, filter_by_name: Optional[list] = None) -> list: """Get a list of all the labels in the organization. Filter by name if desired. Args: - filter_by_name (list, optional): List of subset of labels to return. Defaults to None. + filter_by_name (list, optional): list of subset of labels to return. Defaults to None. Returns: list: [ @@ -316,9 +320,9 @@ def get_or_create_label( def update_label( self, label_name: str, - new_label_name: str = None, - new_color: str = None, - new_show_in_list: bool = None, + new_label_name: Optional[str] = None, + new_color: Optional[str] = None, + new_show_in_list: Optional[bool] = None, ) -> dict: """Update an existing label with the new_* fields. If the new_* fields are not provided, then the existing values are used. @@ -631,12 +635,12 @@ def get_or_create_cycle( # to keep the response consistent add back in the status return selected - def get_cycle_by_name(self, cycle_name: str, set_cycle_id: bool = None) -> dict: + def get_cycle_by_name(self, cycle_name: str, set_cycle_id: bool = False) -> dict: """Set the current cycle by name. Args: cycle_name (str): name of the cycle to set - set_cycle_id (bool): set the cycle_id on the object for later use. Defaults to None. + set_cycle_id (bool): set the cycle_id on the object for later use. Defaults to False. Returns: dict: { @@ -1005,7 +1009,6 @@ def create_extra_data_columns_from_file(self, columns_csv_filepath: str) -> list Args: 'columns_csv_filepath': 'path/to/file' file is expected to have headers: column_name, display_name, column_description, - inventory_type (Property or Taxlot), and data_type (SEED column data_types) See example file at tests/data/test-seed-create-columns.csv @@ -1075,7 +1078,7 @@ def get_meter(self, property_view_id: int, meter_type: str, source: str, source_ else: return None - def get_or_create_meter(self, property_view_id: int, meter_type: str, source: str, source_id: str) -> Optional[Dict[Any, Any]]: + def get_or_create_meter(self, property_view_id: int, meter_type: str, source: str, source_id: str) -> Optional[dict[Any, Any]]: """get or create a meter for a property view. Args: @@ -1151,9 +1154,6 @@ def get_meter_data(self, property_id, interval: str = 'Exact', excluded_meter_id meter_data = self.client.post(endpoint='properties_meter_usage', url_args={"PK": property_id}, json=payload) return meter_data - def save_meter_data(self, property_id: int, meter_id: int, meter_data) -> dict: - pass - def start_save_data(self, import_file_id: int, multiple_cycle_upload: bool = False) -> dict: """start the background process to save the data file to the database. This is the state before the mapping. @@ -1284,6 +1284,118 @@ def check_meters_tab_exist(self, import_file_id: int) -> bool: # if the data is set to True, then return such return response + def get_pm_report_template_names(self, pm_username: str, pm_password: str) -> dict: + """Download the PM report templates. + + Args: + pm_username (str): username for Energystar Portfolio Manager + pm_password (str): password for Energystar Portfolio Manager + + Sample return shown below. + Returns: dict: { + "status": "success", + "templates": [ + { + 'id': 4438244, + 'name': '179D Test', + 'date': '7/03/2023 1:09 PM', + 'timestamp': 1688404158086, + 'hasData': 1, + 'newReport': 0, + 'pending': 0, + 'errored': 0, + 'type': 0, + 'subtype': 4, + 'hasSiteEUIOrWaterUseNAMessages': False, + 'children': [], + 'hasChildrenRows': False, + 'countOfChildrenRows': 0, + 'z_seed_child_row': False, + 'display_name': '179D Test' + } + ], + } + """ + response = self.client.post( + endpoint="portfolio_manager_report_templates", + json={"username": pm_username, "password": pm_password}, + ) + # Return the report templates + return response + + def download_pm_report(self, pm_username: str, pm_password: str, pm_template: dict) -> str: + """Download a PM report. + + Args: + pm_username (str): username for Energystar Portfolio Manager + pm_password (str): password for Energystar Portfolio Manager + pm_template (dict): the full template object dict returned from get_pm_report_template_names + + Sample return shown below. + Returns the path to the report template workbook file + """ + response = self.client.post( + endpoint="portfolio_manager_report", + json={"username": pm_username, + "password": pm_password, + "template": pm_template}, + ) + + # Get the "properties" key from the dictionary. + properties = response["properties"] + + # Create an XLSX workbook object. + workbook = Workbook() + + # Create a sheet object in the workbook. + sheet = workbook.active + + # Get the header row from the API response. + header_row = [] + for property in properties: + for key in property: + if key not in header_row: + header_row.append(key) + + # Write the header row to the sheet object. + if sheet: + sheet.append(header_row) + + # Loop over the list of dictionaries and write the data to the sheet object. + for property in properties: + row = [] + for key in header_row: + row.append(property[key]) + if sheet: + sheet.append(row) + + # Report Template name + report_template_name = pm_template['name'] + + # Filename + file_name = f"{pm_username}_{report_template_name}.xlsx" + + # Folder name + folder_name = "reports" + + if not os.path.exists(folder_name): + os.mkdir(folder_name) + + # Set the file path. + file_path = os.path.join(folder_name, file_name) + + # Save the workbook object. + workbook.save(file_path) + + # Current directory + curdir = os.getcwd() + + # Define the datafile path + datafile_path = os.path.join(curdir, file_path) + + # Return the report templates + return datafile_path + def import_files_reuse_inventory_file_for_meters(self, import_file_id: int) -> dict: """Reuse an import file to create all the meter entries. This method is used for ESPM related data files. The result will be another import_file ID for the @@ -1427,35 +1539,21 @@ def retrieve_at_building_and_update(self, audit_template_building_id: int, cycle return response - def retrieve_at_submission_metadata(self, audit_template_submission_id: int) -> dict: - """Connect to audit template and retrieve audit report json (metadata only) by submission ID - - Args: - audit_template_submission_id (int): ID of the AT submission report (different than building ID) - - Returns: - dict: Response from the SEED API - """ - - # api/v3/audit_template/pk/get_submission - response = self.client.get( - None, - required_pk=False, - endpoint="audit_template_submission", - url_args={"PK": audit_template_submission_id}, - report_format='json' - ) - - return response - - def retrieve_at_submission_and_update(self, audit_template_submission_id: int, cycle_id: int, seed_id: int, report_format: str = 'pdf', filename: str = None) -> dict: + def retrieve_at_submission_and_update( + self, + audit_template_submission_id: int, + cycle_id: int, + seed_id: int, + report_format: str = 'pdf', + filename: Optional[str] = None, + ) -> dict: """Connect to audit template and retrieve audit report by submission ID Args: audit_template_submission_id (int): ID of the AT submission report (different than building ID) cycle_id (int): Cycle ID in SEED (needed for XML but not actually for PDF) seed_id (int): PropertyView ID in SEED - file_format (str): pdf or xml report, defaults to pdf + report_format (str): pdf or xml report, defaults to pdf filename (str): filename to use to upload to SEED Returns: @@ -1527,7 +1625,7 @@ def retrieve_portfolio_manager_property(self, username: str, password: str, pm_p save_file_name (Path): Location to save the file, preferably an absolute path Returns: - bool: Did the file download? + dict: Did the file download? """ if save_file_name.exists(): raise Exception(f"Save filename already exists, save to a new file name: {save_file_name}") @@ -1552,7 +1650,7 @@ def import_portfolio_manager_property(self, seed_id: int, cycle_id: int, mapping seed_id (int): Property view ID to update with the ESPM file cycle_id (int): Cycle ID mapping_profile_id (int): Column Mapping Profile ID - file: path to file downloaded from the retrieve_portfolio_manager_report method above + file_path: path to file downloaded from the retrieve_portfolio_manager_report method above ESPM file will have meter data that we want to handle (electricity and natural gas) in the 'Meter Entries' tab""" diff --git a/pyseed/seed_client_base.py b/pyseed/seed_client_base.py index 9a10599..22a009c 100644 --- a/pyseed/seed_client_base.py +++ b/pyseed/seed_client_base.py @@ -33,21 +33,23 @@ # Constants (Should end with a slash) URLS = { 'v3': { - 'columns': '/api/v3/columns/', 'column_mapping_profiles': '/api/v3/column_mapping_profiles/', 'column_mapping_profiles_filter': '/api/v3/column_mapping_profiles/filter/', + 'columns': '/api/v3/columns/', 'cycles': '/api/v3/cycles/', 'datasets': '/api/v3/datasets/', 'gbr_properties': '/api/v3/gbr_properties/', 'green_assessment': '/api/v3/green_assessments/', 'green_assessment_property': '/api/v3/green_assessment_properties/', 'green_assessment_url': '/api/v3/green_assessment_urls/', + 'import_files': '/api/v3/import_files/', + 'import_files_reuse_inventory_file_for_meters': '/api/v3/import_files/reuse_inventory_file_for_meters/', 'labels': '/api/v3/labels/', 'labels_property': '/api/v3/labels_property/', 'labels_taxlot': '/api/v3/labels_taxlot/', - 'import_files': '/api/v3/import_files/', - 'import_files_reuse_inventory_file_for_meters': '/api/v3/import_files/reuse_inventory_file_for_meters/', 'organizations': '/api/v3/organizations/', + 'portfolio_manager_report': '/api/v3/portfolio_manager/report/', + 'portfolio_manager_report_templates': '/api/v3/portfolio_manager/template_list/', 'properties': '/api/v3/properties/', 'properties_labels': '/api/v3/properties/labels/', 'properties_search': '/api/v3/properties/search/', @@ -58,25 +60,25 @@ # No versioning endpoints 'version': '/api/version/', # POSTs with replaceable keys - 'import_files_start_save_data_pk': '/api/v3/import_files/PK/start_save_data/', + 'import_files_check_meters_tab_exists_pk': '/api/v3/import_files/PK/check_meters_tab_exists/', 'import_files_start_map_data_pk': '/api/v3/import_files/PK/map/', 'import_files_start_matching_pk': '/api/v3/import_files/PK/start_system_matching_and_geocoding/', - 'import_files_check_meters_tab_exists_pk': '/api/v3/import_files/PK/check_meters_tab_exists/', + 'import_files_start_save_data_pk': '/api/v3/import_files/PK/start_save_data/', 'org_column_mapping_import_file': 'api/v3/organizations/ORG_ID/column_mappings/', 'portfolio_manager_property_download': '/api/v3/portfolio_manager/PK/download/', # PUTs with replaceable keys: 'properties_update_with_buildingsync': 'api/v3/properties/PK/update_with_building_sync/', - 'property_update_with_espm': 'api/v3/properties/PK/update_with_espm/', 'properties_upload_inventory_document': 'api/v3/properties/PK/upload_inventory_document', + 'property_update_with_espm': 'api/v3/properties/PK/update_with_espm/', # GETs with replaceable keys 'analyses_views': '/api/v3/analyses/PK/views/ANALYSIS_VIEW_PK/', + 'audit_template_building_xml': '/api/v3/audit_template/PK/get_building_xml', + 'audit_template_submission': '/api/v3/audit_template/PK/get_submission', 'import_files_matching_results': '/api/v3/import_files/PK/matching_and_geocoding_results/', 'progress': '/api/v3/progress/PROGRESS_KEY/', - 'properties_meters': '/api/v3/properties/PK/meters/', - 'properties_meter_usage': '/api/v3/properties/PK/meter_usage/', 'properties_analyses': '/api/v3/properties/PK/analyses/', - 'audit_template_building_xml': '/api/v3/audit_template/PK/get_building_xml', - 'audit_template_submission': '/api/v3/audit_template/PK/get_submission', + 'properties_meter_usage': '/api/v3/properties/PK/meter_usage/', + 'properties_meters': '/api/v3/properties/PK/meters/', # GET & POST with replaceable keys 'properties_meters_reading': '/api/v3/properties/PK/meters/METER_PK/readings/', } diff --git a/requirements-test.txt b/requirements-test.txt index 15cb708..39ec4a5 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,7 @@ -r requirements.txt flake8==4.0.1 mock==4.0.3 -mypy==0.910 +mypy==1.10.0 pre-commit==2.19.0 pytest==7.1.2 pytest-cov==3.0.0 diff --git a/requirements.txt b/requirements.txt index 5b3eaca..c50220a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +openpyxl==3.1.2 requests>=2.28.0 typing==3.6.1 diff --git a/tests/test_seed_client.py b/tests/test_seed_client.py index 1bf2766..baeb588 100644 --- a/tests/test_seed_client.py +++ b/tests/test_seed_client.py @@ -66,6 +66,24 @@ def test_seed_buildings(self): # ESPM test creates a building now too, assert building count is 10 or 11? assert len(buildings) == 10 + def test_get_pm_report_template_names(self): + pm_un = os.environ.get('SEED_PM_UN', False) + pm_pw = os.environ.get('SEED_PM_PW', False) + if not pm_un or not pm_pw: + self.fail(f"Somehow PM test was initiated without {pm_un} or {pm_pw} in the environment") + response = self.seed_client.get_pm_report_template_names(pm_un, pm_pw) + templates = response["templates"] + # loop through the array templates and make a list of all the name keys + template_names = [] + for template in templates: + template_names.append(template["name"]) + assert isinstance(template_names, list) + assert len(template_names) >= 17 + assert "BPS Workflow 2021" in template_names + assert "AT Properties" in template_names + # check that the status is success + assert response["status"] == "success" + def test_search_buildings(self): # set cycle self.seed_client.get_cycle_by_name('pyseed-api-test', set_cycle_id=True)