diff --git a/cid/common.py b/cid/common.py index 4c337477..39646ab0 100644 --- a/cid/common.py +++ b/cid/common.py @@ -194,7 +194,10 @@ def __loadPlugins(self) -> dict: print(f"\t{ep.name} loaded") plugins.update({ep.value: plugin}) try: - self.resources = merge_objects(self.resources, plugin.provides(), depth=1) + resources = plugin.provides() + if ep.value != 'cid.builtin.core': + resources.get('views', {}).pop('account_map', None) # protect account_map from overriding + self.resources = merge_objects(self.resources, resources, depth=1) except AttributeError: logger.warning(f'Failed to load {ep.name}') print('\n') @@ -315,6 +318,7 @@ def load_resource_file(self, source): except Exception as exc: logger.warning(f'Failed to load resources from {source}: {exc}') return + resources.get('views', {}).pop('account_map', None) # Exclude account map as it is a special view self.resources = merge_objects(self.resources, resources, depth=1) def load_catalog(self, catalog_url): diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index afdd3f31..fae3a7c8 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -83,7 +83,7 @@ def cur(self, client) -> CUR: @property def accounts(self) -> dict: - """ Get Accoutns with the right mapping """ + """ Get Accounts with the right mapping """ if self._accounts: # Required keys mapping, used in renaming below key_mapping = { @@ -106,7 +106,7 @@ def accounts(self) -> dict: @lru_cache(1000) def detect_metadata_table(self, name): - """ detect metatable with the list of accounts """ + """ detect meta table with the list of accounts """ cid_print('Autodiscovering metadata table') # FIXME: This will only work for current Athena Database. We might want to check optimization_data base as well @@ -148,9 +148,13 @@ def create(self, name) -> bool: # Query path view_definition = self.athena._resources.get('views').get(name, {}) - view_file = view_definition.get('File') - - template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8')) + if view_definition.get('File'): + view_file = view_definition.get('File') + template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8')) + elif view_definition.get('data'): + template = Template(str(view_definition.get('data'))) + else: + raise CidError(f'{name} definition does not contain File or data: {view_definition}') # Fill in TPLs columns_tpl = { @@ -187,7 +191,7 @@ def get_dummy_account_mapping_sql(self, name) -> list: return compiled_query def get_organization_accounts(self) -> list: - """ Retreive AWS Organization account """ + """ Retrieve AWS Organization account """ # Init clients accounts = [] orgs = self.session.client('organizations') @@ -210,14 +214,14 @@ def get_organization_accounts(self) -> list: return accounts def check_file_exists(self, file_path) -> bool: - """ Checks if the givven file exists """ + """ Checks if the given file exists """ # Set base paths abs_path = Path().absolute() return Path.is_file(abs_path / file_path) def get_csv_accounts(self, file_path) -> list: - """ Retreive accounts from CSV file """ + """ Retrieve accounts from CSV file """ accounts = [ {str(k).lower().replace(" ", "_"): str(v) for k, v in row.items()} for row in read_csv(file_path) @@ -229,7 +233,7 @@ def get_csv_accounts(self, file_path) -> list: def select_metadata_collection_method(self) -> str: """ Selects the method to collect metadata """ logger.info('Metadata source selection') - # Ask user which method to use to retreive account list + # Ask user which method to use to retrieve account list account_map_sources = { 'Dummy (CUR account data, no names)': 'dummy', 'AWS Organizations (one time account listing)': 'organization', diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index 78dc32e2..dea12700 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -158,7 +158,10 @@ def metadata(self) -> dict: if get_parameters().get('cur-table-name'): self._tableName = get_parameters().get('cur-table-name') - self._metadata = self.athena.get_table_metadata(self._tableName) + try: + self._metadata = self.athena.get_table_metadata(self._tableName) + except self.athena.exceptions.EntityNotFoundException as exc: + raise CidCritical('Provided cur-table-name {self._tableName} is not found.') from exc res, message = self.table_is_cur(table=self._metadata, return_reason=True) if not res: raise CidCritical(f'Table {self._tableName} does not look like CUR. {message}') diff --git a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml index d63abf9d..a65f1eb9 100644 --- a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml +++ b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml @@ -141,19 +141,6 @@ datasets: schedules: - default views: - account_map: - dependsOn: - cur: true - data: |- - CREATE OR REPLACE VIEW ${athena_database_name}.account_map AS - SELECT DISTINCT - line_item_usage_account_id account_id - , "max_by"(bill_payer_account_id, line_item_usage_start_date) parent_account_id - , "max_by"(line_item_usage_account_id, line_item_usage_start_date) account_name - , "max_by"(line_item_usage_account_id, line_item_usage_start_date) account_email_id - FROM - "${cur_table_name}" - GROUP BY line_item_usage_account_id data_transfer_view: dependsOn: cur: true