Feature - better status - shows owners and last dataset ingestion

cid/common.py

@@ -108,7 +108,7 @@ def glue(self) -> Glue:
                 'glue': Glue(self.base.session)
             })
         return self._clients.get('glue')
-    
+
     @property
     def organizations(self) -> Organizations:
         if not self._clients.get('organizations'):
@@ -467,7 +467,7 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
                     if not isinstance(ds, Dataset) or ds.name != dataset_name:
                         continue
                     if dashboard_definition.get('templateId'):
-                        # For templates we can additionaly verify dataset fields
+                        # For templates we can additionally verify dataset fields
                         dataset_fields = {col.get('Name'): col.get('Type') for col in ds.columns}
                         src_fields = source_template.datasets.get(ds_map.get(dataset_name, dataset_name) )
                         required_fileds = {col.get('Name'): col.get('DataType') for col in src_fields}
@@ -481,7 +481,7 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
                         else:
                             matching_datasets.append(ds)
                     else:
-                        # for definitions datasets we do not have any possibilty to check if dataset with a given name matches
+                        # for definitions datasets we do not have any possibility to check if dataset with a given name matches
                         matching_datasets.append(ds)
 
                 if not matching_datasets:
@@ -924,25 +924,22 @@ def update(self, dashboard_id, recursive=False, force=False, **kwargs):
 
 
     def check_dashboard_version_compatibility(self, dashboard_id):
-
-        """
-            Returns True | False | None if could not check 
-        """
+        """ Returns True | False | None if could not check """
         try:
             dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
         except CidCritical:
             dashboard = None
         if not dashboard:
             print(f'Dashboard "{dashboard_id}" is not deployed')
             return None
-        if not isinstance(dashboard.deployedTemplate, CidQsTemplate): 
+        if not isinstance(dashboard.deployedTemplate, CidQsTemplate):
             print(f'Dashboard "{dashboard_id}" does not have a versioned template')
             return None
         if not isinstance(dashboard.sourceTemplate, CidQsTemplate):
             print(f"Cannot access QuickSight source template for {dashboard_id}")
             return None
         try:
-            cid_version = dashboard.deployedTemplate.cid_version            
+            cid_version = dashboard.deployedTemplate.cid_version
         except ValueError:
             logger.debug("The cid version of the deployed dashboard could not be retrieved")
             cid_version = "N/A"
@@ -954,32 +951,24 @@ def check_dashboard_version_compatibility(self, dashboard_id):
             cid_version_latest = "N/A"
 
         if dashboard.latest:
-            print("You are up to date!")       
-            print(f"  CID Version      {cid_version}")
-            print(f"  TemplateVersion  {dashboard.deployed_version} ")
-
-            logger.debug("The dashboard is up-to-date")
-            logger.debug(f"CID Version      {cid_version}")
-            logger.debug(f"TemplateVersion  {dashboard.deployed_version} ")
+            cid_print("You are up to date!")
+            cid_print(f"  Version    {cid_version}")
+            cid_print(f"  VersionId  {dashboard.deployed_version} ")
         else:
-            print(f"An update is available:")
-            print("                   Deployed -> Latest")
-            print(f"  CID Version      {str(cid_version): <9}   {str(cid_version_latest): <6}")
-            print(f"  TemplateVersion  {str(dashboard.deployedTemplate.version): <9}   {dashboard.latest_version: <6}")
-
-            logger.debug("An update is available")
-            logger.debug(f"CID Version      {str(cid_version): <9} --> {str(cid_version_latest): <6}")
-            logger.debug(f"TemplateVersion  {str(dashboard.deployedTemplate.version): <9} -->  {dashboard.latest_version: <6}")
+            cid_print(f"An update is available:")
+            cid_print("              Deployed -> Latest")
+            cid_print(f"  Version    {str(cid_version): <9}   {str(cid_version_latest): <9}")
+            cid_print(f"  VersionId  {str(dashboard.deployedTemplate.version): <9}   {dashboard.latest_version: <9}")
 
         # Check if version are compatible
         compatible = None
         try:
             compatible = dashboard.sourceTemplate.cid_version.compatible_versions(dashboard.deployedTemplate.cid_version)
-        except ValueError as e:
-            logger.info(e)
-            
+        except ValueError as exc:
+            logger.info(exc)
+
         return compatible
-    
+
     def update_dashboard(self, dashboard_id, dashboard_definition):
 
         dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
@@ -996,8 +985,7 @@ def update_dashboard(self, dashboard_id, dashboard_definition):
             print(f"Latest template: {dashboard.sourceTemplate.arn}/version/{dashboard.latest_version}")
         else:
             print('Unable to determine dashboard source.')
-
-
+
         if dashboard.status == 'legacy':
             if get_parameter(
                 param_name=f'confirm-update',
@@ -1016,7 +1004,7 @@ def update_dashboard(self, dashboard_id, dashboard_definition):
         # Update dashboard
         print(f'\nUpdating {dashboard_id}')
         logger.debug(f"Updating {dashboard_id}")
-        
+
         try:
             self.qs.update_dashboard(dashboard, dashboard_definition)
             print('Update completed\n')

cid/helpers/quicksight/__init__.py

@@ -15,7 +15,7 @@
 from cid.helpers.quicksight.dataset import Dataset
 from cid.helpers.quicksight.datasource import Datasource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
-from cid.utils import get_parameter, get_parameters, exec_env, cid_print
+from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago
 from cid.exceptions import CidCritical, CidError
 
 logger = logging.getLogger(__name__)
@@ -549,7 +549,7 @@ def discover_data_sources(self) -> None:
         except Exception as exc:
             logger.debug(exc, exc_info=True)
 
-    def discover_dashboards(self, display: bool=False, refresh: bool=False) -> None:
+    def discover_dashboards(self, refresh: bool=False) -> None:
         """ Discover deployed dashboards """
         if refresh or self._dashboards is None:
             self._dashboards = {}
@@ -572,15 +572,6 @@ def discover_dashboards(self, display: bool=False, refresh: bool=False) -> None:
                 self.discover_dashboard(dashboard_id)
                 # Update progress bar
                 bar.update(0, 'Complete')
-        if not display:
-            return
-        for dashboard in self._dashboards.values():
-            if dashboard.health:
-                health = 'healthy'
-            else:
-                health = 'unhealthy'
-            print(f'\t{dashboard.name} ({dashboard.id}, {health}, {dashboard.status})')
-
 
     def list_dashboards(self) -> list:
         parameters = {
@@ -792,7 +783,7 @@ def describe_dashboard(self, poll: bool=False, **kwargs) -> Union[None, Dashboar
                     time.sleep(5)
                     continue
                 logger.debug(response)
-                dashboard = Dashboard(response)
+                dashboard = Dashboard(response, qs=self)
                 current_status = dashboard.version.get('Status')
                 if not poll:
                     break
@@ -906,6 +897,33 @@ def describe_dataset(self, id, timeout: int=1) -> Dataset:
 
         return self._datasets.get(id, None)
 
+    def get_dataset_last_ingestion(self, dataset_id) -> str:
+        """returns human friendly status of the latest ingestion"""
+        try:
+            ingestions = self.client.list_ingestions(
+                DataSetId=dataset_id,
+                AwsAccountId=self.account_id,
+            ).get('Ingestions', [])
+        except self.client.exceptions.ResourceNotFoundException:
+            return '<RED>NotFound<END>'
+        except self.client.exceptions.AccessDeniedException:
+            return '<YELLOW>AccessDenied<END>'
+        if not ingestions:
+            return None
+        last_ingestion = ingestions[0] # Suppose it is the latest
+        status = last_ingestion.get('IngestionStatus')
+        time_ago = ago(last_ingestion.get('CreatedTime'))
+        if last_ingestion.get('ErrorInfo', {}).get('Type') == "DATA_SET_NOT_SPICE":
+            return '<BLUE>DIRECT_QUERY<END>'
+        if status in ('COMPLETED',):
+            status = f'<GREEN>{status}<END>'
+            time_in_mins = int(int(last_ingestion.get('IngestionTimeInSeconds', 0) or 0) / 60)
+            return f"{status} ({time_in_mins} mins, {last_ingestion['RowInfo']['RowsIngested']} rows) {time_ago}"
+        if status in ('FAILED', 'CANCELLED'):
+            status = f'<RED>{status}<END>'
+            return f"{status} ({last_ingestion['ErrorInfo']['Type']} {last_ingestion['ErrorInfo']['Message']}) {time_ago}"
+        return f'{status} {time_ago}'
+
     def discover_datasets(self, _datasets: list=None):
         """ Discover datasets in the account """
 
@@ -1347,6 +1365,10 @@ def update_template_permissions(self, **update_parameters):
         logger.debug(update_status)
         return update_status
 
+    def get_dashboard_permissions(self, dashboard_id):
+        """ get_dashboard_permissions """
+        return self.client.describe_dashboard_permissions(AwsAccountId=self.account_id, DashboardId=dashboard_id)['Permissions']
+
     def dataset_diff(self, raw1, raw2):
         """ get dataset diff """
         return diff(

cid/helpers/quicksight/dashboard.py

@@ -1,18 +1,18 @@
 import click
 import json
 import logging
-import os
 from typing import Dict
 
 from cid.helpers.quicksight.resource import CidQsResource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
-from cid.utils import is_unattendent_mode
+from cid.utils import cid_print, get_yesno_parameter
+from cid.helpers.quicksight.resource import CidQsResource
 
 logger = logging.getLogger(__name__)
 
 
 class Dashboard(CidQsResource):
-    def __init__(self, raw: dict) -> None:
+    def __init__(self, raw: dict, qs=None) -> None:
         super().__init__(raw)
         # Initialize properties
         self.datasets: Dict[str, str] = {}
@@ -22,6 +22,7 @@ def __init__(self, raw: dict) -> None:
         self.status_detail = str()
         # Source template in origin account
         self.sourceTemplate: CidQsTemplate = None
+        self.qs = qs
 
     @property
     def id(self) -> str:
@@ -100,53 +101,59 @@ def status(self) -> str:
         return self._status
 
     def display_status(self) -> None:
-
-        print('\nDashboard status:')
-        print(f"  Name (id): {self.name} ({self.id})")
-        print(f"  Status: {self.status}")
-        print(f"  Health: {'healthy' if self.health else 'unhealthy'}")
-
+        """Display status of dashboard"""
+
+        cid_print('\n<BOLD>Dashboard status:<END>')
+        cid_print(f"  <BOLD>Id:<END>        {self.id}")
+        cid_print(f"  <BOLD>Name:<END>      {self.name}")
+        cid_print(f"  <BOLD>Health:<END>    {'<GREEN>healthy<END>' if self.health else '<RED>unhealthy<END>'}")
+        cid_print(f"  <BOLD>Status:<END>    {'<GREEN>' + self.status + '<END>' if self.health else '<RED>' + self.status + '<END>'}")
+
         if self.status_detail:
-            print(f"  Status detail: {self.status_detail}")
-            
-        cid_version = None
-        cid_version_latest = None
-        
+            cid_print(f"  <BOLD>Status detail:<END> {self.status_detail}")
+
+        cid_version = "N/A"
+        cid_version_latest =  "N/A"
+
         try:
             cid_version = self.deployedTemplate.cid_version
         except ValueError:
             logger.debug("The cid version of the deployed dashboard could not be retrieved")
-            cid_version = "N/A"
 
         try:
-            cid_version_latest = self.sourceTemplate.cid_version if isinstance(self.sourceTemplate, CidQsTemplate) else "N/A"
+            if isinstance(self.sourceTemplate, CidQsTemplate):
+                cid_version_latest = self.sourceTemplate.cid_version
         except ValueError:
             logger.debug("The latest version of the dashboard could not be retrieved")
-            cid_version_latest = "N/A"
-            print(f"  CID Version     {cid_version}")
-            print(f"  TemplateVersion {self.deployed_version} ")
 
         if self.latest:
-            logger.debug("The dashboard is up-to-date")
-            logger.debug(f"CID Version {cid_version}")
-            logger.debug(f"TemplateVersion {self.deployed_version} ")
+            cid_print(f"  <BOLD>Version:<END>   <GREEN>{cid_version}<END> (latest)")
+            cid_print(f"  <BOLD>VersionId:<END> <GREEN>{self.deployed_version}<END> (latest)")
         else:
-            print(f"  CID Version     {str(cid_version): <6} --> {str(cid_version_latest): <6}")
-            print(f"  TemplateVersion {str(self.deployed_version): <6} --> {str(self.latest_version): <6}")
-
             logger.debug("An update is available")
-            logger.debug(f"CID Version      {str(cid_version): <9} --> {str(cid_version_latest): <6}")
-            logger.debug(f"TemplateVersion  {str(self.deployed_version): <9} -->  {str(self.latest_version): <6}")
-
+            cid_print(f"  <BOLD>Version:<END>   <YELLOW>{str(cid_version): <8} --> {str(cid_version_latest): <8}<END>")
+            cid_print(f"  <BOLD>VersionId:<END> <YELLOW>{str(self.deployed_version): <8} --> {str(self.latest_version): <8}<END>")
+
+        cid_print('  <BOLD>Owners:<END>')
+        try:
+            permissions = self.qs.get_dashboard_permissions(self.id)
+            for permission in permissions:
+                if 'quicksight:UpdateDashboardPermissions' in permission["Actions"]:
+                    cid_print('    ' + permission["Principal"].split('user/default/')[-1])
+        except Exception as exc:
+            if "AccessDenied" in str(exc):
+                cid_print('     <RED>AccessDenied<END>')
+
         if self.datasets:
-            print(f"  Datasets: {', '.join(sorted(self.datasets.keys()))}")
+            cid_print(f"  <BOLD>Datasets:<END>")
+            for dataset_name, dataset_id in  sorted(self.datasets.items()):
+                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>'
+                cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) {status}')
+
         print('\n')
-        if click.confirm('Display dashboard raw data?'):
+        if get_yesno_parameter('display-raw', 'Display dashboard raw data?', default='yes'):
             print(json.dumps(self.raw, indent=4, sort_keys=True, default=str))
-   
+
     def display_url(self, url_template: str, launch: bool = False, **kwargs) -> None:
         url = url_template.format(dashboard_id=self.id, **kwargs)
         print(f"#######\n####### {self.name} is available at: " + url + "\n#######")
-        _supported_env = os.environ.get('AWS_EXECUTION_ENV') not in ['CloudShell', 'AWS_Lambda']
-        if _supported_env and not is_unattendent_mode() and launch and click.confirm('Do you wish to open it in your browser?'):
-            click.launch(url)

cid/utils.py

@@ -1,8 +1,10 @@
 import os
 import sys
+import math
 import inspect
 import logging
 import platform
+import datetime
 from typing import Any, Dict
 from functools import lru_cache as cache
 from collections.abc import Iterable
@@ -166,7 +168,7 @@ def set_parameters(parameters: dict, all_yes: bool=None) -> None:
         global _all_yes
         _all_yes = all_yes
 
-def is_unattendent_mode() -> bool:
+def is_unattended_mode() -> bool:
     return _all_yes
 
 def get_parameters():
@@ -258,3 +260,20 @@ def unset_parameter(param_name):
         del params[param_name]
         logger.info(f'Cleared {param_name}={value}, from parameters')
 
+
+def ago(time):
+    """ Calculate a '3 hours ago' type string from a python datetime.
+    credits: https://gist.github.com/tonyblundell/2652369
+    """
+    units = {
+        'days': lambda diff: diff.days,
+        'hours': lambda diff: diff.seconds / 3600,
+        'minutes': lambda diff: diff.seconds % 3600 / 60,
+    }
+    diff = datetime.datetime.now().replace(tzinfo=time.tzinfo) - time
+    for unit in units:
+        dur = math.floor(units[unit](diff)) # Run the lambda function to get a duration
+        if dur > 0:
+            unit = unit[:-dur] if dur == 1 else unit # De-pluralize if duration is 1 ('1 day' vs '2 days')
+            return '%s %s ago' % (dur, unit)
+    return 'just now'