Feature crawler

README.md

@@ -83,21 +83,28 @@ cid-cmd share
 ```
 
 #### Initialize Amazon QuickSight
-One time action to intialize Amazon QuickSight Enerprise Edition.
+One time action to initialize Amazon QuickSight Enterprise Edition.
 
 ```bash
-cid-cmd initqs
+cid-cmd init-qs
 ```
 
+#### Initialize CUR
+One time action to initialize Athena table and Crawler from s3 with CUR data.
+
+```bash
+cid-cmd init-cur
+```
 
 #### Delete Dashboard and all dependencies unused by other
 Delete Dashboards and all dependencies unused by other CID-managed dashboards.(including QuickSight datasets, Athena views and tables)
 ```bash
 cid-cmd delete
 ```
+
 #### Delete Command Options:
 ```
- --dashboard-id TEXT QuickSight dashboard id 
+ --dashboard-id TEXT QuickSight dashboard id
  --athena-database TEXT Athena database
  ```
 

cfn-templates/cid-cfn.yml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v0.2.42
+Description: Deployment of Cloud Intelligence Dashboards v0.3.0
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -1506,7 +1506,7 @@ Resources:
       Description: An AWS managed layer with a cid-cmd package installed
       Content:
         S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}'
-        S3Key: 'cid-resource-lambda-layer/cid-0.2.42.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-0.3.0.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
 

cid/_version.py

@@ -1,2 +1 @@
-
-__version__ = '0.2.42'
+__version__ = '0.3.0'

cid/builtin/core/data/queries/shared/cur.yaml

@@ -0,0 +1,89 @@
+DatabaseName: "${athena_database_name}"
+TableInput:
+  Name: "${athenaTableName}"
+  Owner: owner
+  Retention: 0
+  TableType: EXTERNAL_TABLE
+  Parameters:
+    compressionType: none
+    classification: parquet
+    UPDATED_BY_CRAWLER: CidCurCrawler # Hard coded Crawler Name
+  StorageDescriptor:
+    BucketColumns: []
+    Compressed: false
+    Location: "${location}"
+    NumberOfBuckets: -1
+    InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+    OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
+    SerdeInfo:
+      Parameters:
+        serialization.format: '1'
+      SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
+    StoredAsSubDirectories: false
+    Columns: # All fields required for CID
+      - {"Name": "bill_bill_type", "Type": "string" }
+      - {"Name": "bill_billing_entity", "Type": "string" }
+      - {"Name": "bill_billing_period_end_date", "Type": "timestamp" }
+      - {"Name": "bill_billing_period_start_date", "Type": "timestamp" }
+      - {"Name": "bill_invoice_id", "Type": "string" }
+      - {"Name": "bill_payer_account_id", "Type": "string" }
+      - {"Name": "identity_line_item_id", "Type": "string" }
+      - {"Name": "identity_time_interval", "Type": "string" }
+      - {"Name": "line_item_availability_zone", "Type": "string" }
+      - {"Name": "line_item_legal_entity", "Type": "string" }
+      - {"Name": "line_item_line_item_description", "Type": "string" }
+      - {"Name": "line_item_line_item_type", "Type": "string" }
+      - {"Name": "line_item_operation", "Type": "string" }
+      - {"Name": "line_item_product_code", "Type": "string" }
+      - {"Name": "line_item_resource_id", "Type": "string" }
+      - {"Name": "line_item_unblended_cost", "Type": "double" }
+      - {"Name": "line_item_usage_account_id", "Type": "string" }
+      - {"Name": "line_item_usage_amount", "Type": "double" }
+      - {"Name": "line_item_usage_end_date", "Type": "timestamp" }
+      - {"Name": "line_item_usage_start_date", "Type": "timestamp" }
+      - {"Name": "line_item_usage_type", "Type": "string" }
+      - {"Name": "pricing_lease_contract_length", "Type": "string" }
+      - {"Name": "pricing_offering_class", "Type": "string" }
+      - {"Name": "pricing_public_on_demand_cost", "Type": "double" }
+      - {"Name": "pricing_purchase_option", "Type": "string" }
+      - {"Name": "pricing_term", "Type": "string" }
+      - {"Name": "pricing_unit", "Type": "string" }
+      - {"Name": "product_cache_engine", "Type": "string" }
+      - {"Name": "product_current_generation", "Type": "string" }
+      - {"Name": "product_database_engine", "Type": "string" }
+      - {"Name": "product_deployment_option", "Type": "string" }
+      - {"Name": "product_from_location", "Type": "string" }
+      - {"Name": "product_group", "Type": "string" }
+      - {"Name": "product_instance_type", "Type": "string" }
+      - {"Name": "product_instance_type_family", "Type": "string" }
+      - {"Name": "product_license_model", "Type": "string" }
+      - {"Name": "product_operating_system", "Type": "string" }
+      - {"Name": "product_physical_processor", "Type": "string" }
+      - {"Name": "product_processor_features", "Type": "string" }
+      - {"Name": "product_product_family", "Type": "string" }
+      - {"Name": "product_product_name", "Type": "string" }
+      - {"Name": "product_region", "Type": "string" }
+      - {"Name": "product_servicecode", "Type": "string" }
+      - {"Name": "product_storage", "Type": "string" }
+      - {"Name": "product_tenancy", "Type": "string" }
+      - {"Name": "product_to_location", "Type": "string" }
+      - {"Name": "product_volume_api_name", "Type": "string" }
+      - {"Name": "product_volume_type", "Type": "string" }
+      - {"Name": "reservation_amortized_upfront_fee_for_billing_period", "Type": "double" }
+      - {"Name": "reservation_effective_cost", "Type": "double" }
+      - {"Name": "reservation_end_time", "Type": "string" }
+      - {"Name": "reservation_reservation_a_r_n", "Type": "string" }
+      - {"Name": "reservation_start_time", "Type": "string" }
+      - {"Name": "reservation_unused_amortized_upfront_fee_for_billing_period", "Type": "double" }
+      - {"Name": "reservation_unused_recurring_fee", "Type": "double" }
+      - {"Name": "savings_plan_amortized_upfront_commitment_for_billing_period", "Type": "double" }
+      - {"Name": "savings_plan_end_time", "Type": "string" }
+      - {"Name": "savings_plan_offering_type", "Type": "string" }
+      - {"Name": "savings_plan_payment_option", "Type": "string" }
+      - {"Name": "savings_plan_purchase_term", "Type": "string" }
+      - {"Name": "savings_plan_savings_plan_a_r_n", "Type": "string" }
+      - {"Name": "savings_plan_savings_plan_effective_cost", "Type": "double" }
+      - {"Name": "savings_plan_start_time", "Type": "string" }
+      - {"Name": "savings_plan_total_commitment_to_date", "Type": "double" }
+      - {"Name": "savings_plan_used_commitment", "Type": "double" }
+  PartitionKeys: ${partitions} # can be a list

cid/builtin/core/data/resources.yaml

@@ -623,6 +623,19 @@ views:
         - aws_accounts
   ta_descriptions:
     File: shared/ta_descriptions.sql
+  cur:
+    type: Glue_Table
+    File: shared/cur.yaml
+    crawler: cur
+    parameters:
+      partitions:
+        type: cur.partitions
+        description: 'comma separated list of CUR partitions ex: ["source_account_id","cur_name_1","cur_name_2","year","month"] ex2: ["year","month"]'
+        default: '[{"Name":"source_account_id","Type":"string"},{"Name":"cur_name_1","Type":"string"},{"Name":"cur_name_2","Type":"string"},{"Name":"year","Type":"string"},{"Name":"month","Type":"string"}]'
+      location:
+        type: cur.location
+        description: 's3 path'
+        default: 's3://cid-{account_id}-shared/cur/'
 
 # Refresh Schedules for QuickSight DataSets
 schedules:
@@ -632,3 +645,45 @@ schedules:
       Interval: DAILY
       TimeOfTheDay: '02:00-05:00'
     RefreshType: FULL_REFRESH
+
+crawlers:
+  cur:
+    data:
+      Name:  'CidCmdCurCrawler'
+      Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
+      Role:  ${crawler_role_arn}
+      DatabaseName: "${athena_database_name}"
+      Targets:
+        S3Targets:
+          - Path: ${location}
+            Exclusions:
+              - '**.json'
+              - '**.yml'
+              - '**.sql'
+              - '**.csv'
+              - '**.csv.metadata'
+              - '**.gz'
+              - '**.zip'
+              - '**/cost_and_usage_data_status/*'
+              - 'aws-programmatic-access-test-object'
+      SchemaChangePolicy:
+        DeleteBehavior: LOG
+      RecrawlPolicy:
+        RecrawlBehavior: CRAWL_EVERYTHING
+      Schedule: cron(0 2 * * ? *)
+      Configuration: |
+        {
+          "Version":1.0,
+          "Grouping": {
+            "TableGroupingPolicy": "CombineCompatibleSchemas"
+          },
+          "CrawlerOutput":{
+            "Tables":{
+              "AddOrUpdateBehavior":"MergeNewColumns"
+            }
+          }
+        }
+    parameters:
+      s3path:
+        default: 's3://cid-{account_id}-cur/cur/'
+        description: CUR path

cid/cli.py

@@ -227,23 +227,48 @@ def cleanup(ctx, **kwargs):
 @cid_command
 def share(ctx, dashboard_id, **kwargs):
     """Share QuickSight resources (Dashboard, Datasets, DataSource)"""
-
     ctx.obj.share(dashboard_id)
 
 @click.option('-v', '--verbose', count=True)
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @cid_command
-def initqs(ctx, **kwargs):
+def init_qs(ctx, **kwargs):
     """Initialize Amazon QuickSight
 
     \b
-
      --enable-quicksight-enterprise (yes|no) Confirm the activation of QuickSight
-     --account-name NAME                     Unique QuickSight account name (Unique across all AWS users) 
+     --account-name NAME                     Unique QuickSight account name (Unique across all AWS users)
      --notification-email EMAIL              User's email for QuickSight notifications
     """
 
-    ctx.obj.initqs(**kwargs)
+    ctx.obj.init_qs(**kwargs)
+
+@click.option('-v', '--verbose', count=True)
+@cid_command
+def init_cur(ctx, **kwargs):
+    """Initialize CUR table
+
+    \b
+     --view-cur-location  s3://BUCKET/PATH   S3 path with CUR data. We support only 2 types of CUR path: 's3://{bucket}/cur' and 's3://{bucket}/{prefix}/{name}/{name}'
+     --crawler-role       ROLE               Name or ARN of crawler role
+    """
+
+    ctx.obj.init_cur(**kwargs)
+
+@click.option('-v', '--verbose', count=True)
+@click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
+@cid_command
+def teardown(ctx, **kwargs):
+    """Delete all CID assets
+
+    \b
+
+    THIS IS VERY DANGEROUS. DO NOT USE IT.
+    """
+
+    ctx.obj.teardown(**kwargs)
 
 if __name__ == '__main__':
     main()
+
+

cid/commands/init_qs.py

@@ -24,27 +24,27 @@ def __init__(self, cid, **kwargs):
         self.cid = cid
 
     def execute(self, *args, **kwargs):
-        """Execute the initilization"""
+        """Execute the initialization"""
         self._create_quicksight_enterprise_subscription()
 
     def _create_quicksight_enterprise_subscription(self):
         """Enable QuickSight Enterprise if not enabled already"""
-        cid_print('Analysing QuickSight Status')
+        cid_print('Analyzing QuickSight Status')
         if self.cid.qs.edition(fresh=True) in ('ENTERPRISE', 'ENTERPRISE_AND_Q'):
             cid_print(f'QuickSight Edition is {self.cid.qs.edition()}')
             return
 
         cid_print(
-            '<BOLD><RED>IMPORTANT<END>: <BOLD>Amazion QuickSight Enterprise Edition is required for Cost Intelligence Dashboard. '
+            '<BOLD><RED>IMPORTANT<END>: <BOLD>Amazon QuickSight Enterprise Edition is required for Cost Intelligence Dashboard. '
             'This will lead to costs in your AWS account (https://aws.amazon.com/quicksight/pricing/).<END>'
         )
 
         if not self.cid.all_yes and not get_yesno_parameter(
                 param_name='enable-quicksight-enterprise',
-                message='Please, confirm enabling of Amazion QuickSight Enterprise',
+                message='Please, confirm enabling of Amazon QuickSight Enterprise',
                 default='no'
             ):
-            cid_print('\tInitalization cancelled')
+            cid_print('\tInitialization cancelled')
             return
 
         for counter in range(MAX_ITERATIONS):
@@ -74,7 +74,7 @@ def _create_quicksight_enterprise_subscription(self):
         cid_print(f'\tQuickSight Edition is {self.cid.qs.edition()}.')
 
     def _get_account_name_for_quicksight(self):
-        """Get the account name for quicksight"""        
+        """Get the account name for quicksight"""
         for _ in range(MAX_ITERATIONS):
             account_name = get_parameter(
                 'account-name',