fix account map creation (#689)

cid/common.py

@@ -194,7 +194,10 @@ def __loadPlugins(self) -> dict:
             print(f"\t{ep.name} loaded")
             plugins.update({ep.value: plugin})
             try:
-                self.resources = merge_objects(self.resources, plugin.provides(), depth=1)
+                resources = plugin.provides()
+                if ep.value != 'cid.builtin.core':
+                    resources.get('views', {}).pop('account_map', None) # protect account_map from overriding
+                self.resources = merge_objects(self.resources, resources, depth=1)
             except AttributeError:
                 logger.warning(f'Failed to load {ep.name}')
         print('\n')
@@ -315,6 +318,7 @@ def load_resource_file(self, source):
         except Exception as exc:
             logger.warning(f'Failed to load resources from {source}: {exc}')
             return
+        resources.get('views', {}).pop('account_map', None) # Exclude account map as it is a special view
         self.resources = merge_objects(self.resources, resources, depth=1)
 
     def load_catalog(self, catalog_url):

cid/helpers/account_map.py

@@ -83,7 +83,7 @@ def cur(self, client) -> CUR:
 
     @property
     def accounts(self) -> dict:
-        """ Get Accoutns with the right mapping """
+        """ Get Accounts with the right mapping """
         if self._accounts:
             # Required keys mapping, used in renaming below
             key_mapping = {
@@ -106,7 +106,7 @@ def accounts(self) -> dict:
 
     @lru_cache(1000)
     def detect_metadata_table(self, name):
-        """ detect metatable with the list of accounts """
+        """ detect meta table with the list of accounts """
         cid_print('Autodiscovering metadata table')
 
         # FIXME: This will only work for current Athena Database. We might want to check optimization_data base as well
@@ -148,9 +148,13 @@ def create(self, name) -> bool:
 
             # Query path
             view_definition = self.athena._resources.get('views').get(name, {})
-            view_file = view_definition.get('File')
-
-            template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8'))
+            if view_definition.get('File'):
+                view_file = view_definition.get('File')
+                template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8'))
+            elif view_definition.get('data'):
+                template = Template(str(view_definition.get('data')))
+            else:
+                raise CidError(f'{name} definition does not contain File or data: {view_definition}')
 
             # Fill in TPLs
             columns_tpl = {
@@ -187,7 +191,7 @@ def get_dummy_account_mapping_sql(self, name) -> list:
         return compiled_query
 
     def get_organization_accounts(self) -> list:
-        """ Retreive AWS Organization account """
+        """ Retrieve AWS Organization account """
         # Init clients
         accounts = []
         orgs = self.session.client('organizations')
@@ -210,14 +214,14 @@ def get_organization_accounts(self) -> list:
         return accounts
 
     def check_file_exists(self, file_path) -> bool:
-        """ Checks if the givven file exists """
+        """ Checks if the given file exists """
         # Set base paths
         abs_path = Path().absolute()
 
         return Path.is_file(abs_path / file_path)
 
     def get_csv_accounts(self, file_path) -> list:
-        """ Retreive accounts from CSV file """
+        """ Retrieve accounts from CSV file """
         accounts = [
             {str(k).lower().replace(" ", "_"): str(v) for k, v in row.items()}
             for row in read_csv(file_path)
@@ -229,7 +233,7 @@ def get_csv_accounts(self, file_path) -> list:
     def select_metadata_collection_method(self) -> str:
         """ Selects the method to collect metadata """
         logger.info('Metadata source selection')
-        # Ask user which method to use to retreive account list
+        # Ask user which method to use to retrieve account list
         account_map_sources = {
             'Dummy (CUR account data, no names)': 'dummy',
             'AWS Organizations (one time account listing)': 'organization',

cid/helpers/cur.py

@@ -158,7 +158,10 @@ def metadata(self) -> dict:
 
         if get_parameters().get('cur-table-name'):
             self._tableName = get_parameters().get('cur-table-name')
-            self._metadata = self.athena.get_table_metadata(self._tableName)
+            try:
+                self._metadata = self.athena.get_table_metadata(self._tableName)
+            except self.athena.exceptions.EntityNotFoundException as exc:
+                raise CidCritical('Provided cur-table-name {self._tableName} is not found.') from exc
             res, message = self.table_is_cur(table=self._metadata, return_reason=True)
             if not res:
                 raise CidCritical(f'Table {self._tableName} does not look like CUR. {message}')

dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml

@@ -141,19 +141,6 @@ datasets:
     schedules:
     - default
 views:
-  account_map:
-    dependsOn:
-      cur: true
-    data: |-
-      CREATE OR REPLACE VIEW ${athena_database_name}.account_map AS
-      SELECT DISTINCT
-        line_item_usage_account_id account_id
-      , "max_by"(bill_payer_account_id, line_item_usage_start_date) parent_account_id
-      , "max_by"(line_item_usage_account_id, line_item_usage_start_date) account_name
-      , "max_by"(line_item_usage_account_id, line_item_usage_start_date) account_email_id
-      FROM
-        "${cur_table_name}"
-      GROUP BY line_item_usage_account_id
   data_transfer_view:
     dependsOn:
       cur: true