Added the optmize reports policy (#695)

cloud_governance/common/clouds/aws/resource_explorer/resource_explorer_operations.py

@@ -0,0 +1,52 @@
+import boto3
+
+from cloud_governance.common.logger.init_logger import logger
+
+# @Todo, This class will be used in the feature releases.
+# @Todo, it helps in find the resource data like tags by using search query
+
+
+class ResourceExplorerOperations:
+    """
+    This class performs the resource explorer operations
+    """
+
+    def __init__(self):
+        self.__client = self.__set_client()
+
+    def __set_client(self):
+        view = self.list_views()
+        region = 'us-east-1'
+        if view:
+            region = view.split(':')[3]
+        return boto3.client('resource-explorer-2', region_name=region)
+
+    def __search(self, search_string: str):
+        try:
+            response = self.__client.search(QueryString=search_string)
+            return response.get('Resources', [])
+        except Exception as err:
+            logger.error(err)
+            return []
+
+    def find_resource_tags(self, resource_id: str):
+        search_results = self.__search(search_string=f'"{resource_id}"')
+        tags = []
+        for resource in search_results:
+            if resource_id in resource.get('Arn'):
+                if resource.get('Properties'):
+                    tags = resource.get('Properties', {})[0].get('Data')
+        return tags
+
+    def list_views(self):
+        """
+        This method returns list the views
+        :return:
+        :rtype:
+        """
+        client = boto3.client('resource-explorer-2', region_name='us-east-1')
+        views = client.list_views()['Views']
+        if views:
+            return views[0]
+        else:
+            raise Exception("No Resource Explorer view found in Region: us-east-1, create one on Free of Charge")

cloud_governance/common/clouds/aws/support/support_operations.py

@@ -0,0 +1,44 @@
+import boto3
+
+from cloud_governance.common.logger.init_logger import logger
+
+
+class SupportOperations:
+    """
+    This class performs the support operations
+    """
+
+    def __init__(self):
+        self.__client = boto3.client('support', region_name='us-east-1')
+
+    def get_describe_trusted_advisor_checks(self):
+        """
+        This method returns the trusted advisor check results
+        :return:
+        :rtype:
+        """
+        try:
+            response = self.__client.describe_trusted_advisor_checks(language='en')
+            return response.get('checks', [])
+        except Exception as err:
+            logger.error(err)
+        return []
+
+    def get_trusted_advisor_reports(self):
+        """
+        This method returns the reports of the checks
+        :return:
+        :rtype:
+        """
+        result = {}
+        try:
+            advisor_checks_list = self.get_describe_trusted_advisor_checks()
+            for check in advisor_checks_list:
+                response = self.__client.describe_trusted_advisor_check_result(checkId=check.get('id'))
+                result.setdefault(check.get('category'), {}).setdefault(check.get('id'), {
+                    'metadata': check,
+                    'reports': response.get('result', [])
+                })
+        except Exception as err:
+            logger.err(err)
+        return result

cloud_governance/common/clouds/aws/utils/common_methods.py

@@ -0,0 +1,28 @@
+def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str',
+                            default_value: any = '') -> any:
+    """
+    This method returns the tag value inputted by tag_name
+    :param tags:
+    :type tags:
+    :param tag_name:
+    :type tag_name:
+    :param cast_type:
+    :type cast_type:
+    :param default_value:
+    :type default_value:
+    :return:
+    :rtype:
+    """
+    if tags:
+        for tag in tags:
+            key = tag.get('Key').lower().replace("_", '').replace("-", '').strip()
+            if key == tag_name.lower():
+                if cast_type:
+                    if cast_type == 'int':
+                        return int(tag.get('Value').split()[0].strip())
+                    elif cast_type == 'float':
+                        return float(tag.get('Value').strip())
+                    else:
+                        return str(tag.get('Value').strip())
+                return tag.get('Value').strip()
+    return default_value

cloud_governance/common/elasticsearch/elasticsearch_operations.py

@@ -311,10 +311,16 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs):
                 if kwargs.get('id'):
                     item['_id'] = item.get(kwargs.get('id'))
                 if not item.get('timestamp'):
-                    item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d")
+                    if 'CurrentDate' in item:
+                        item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d")
+                    else:
+                        item['timestamp'] = datetime.utcnow()
                 item['_index'] = index
-                item['AccountId'] = str(item.get('AccountId'))
-                item['Policy'] = self.__environment_variables_dict.get('policy')
+                if item.get('AccountId'):
+                    item['AccountId'] = str(item.get('AccountId'))
+                if 'account' not in item:
+                    item['account'] = self.__account
+                item['policy'] = self.__environment_variables_dict.get('policy')
             response = bulk(self.__es, bulk_items)
             if response:
                 total_uploaded += len(bulk_items)

cloud_governance/main/environment_variables.py

@@ -53,7 +53,7 @@ def __init__(self):
                                                                         'empty_roles', 'ip_unattached',
                                                                         'unused_nat_gateway',
                                                                         'zombie_snapshots', 'skipped_resources',
-                                                                        'monthly_report']
+                                                                        'monthly_report', 'optimize_resources_report']
         es_index = 'cloud-governance-policy-es-index'
         self._environment_variables_dict['cost_policies'] = ['cost_explorer', 'cost_over_usage', 'cost_billing_reports',
                                                              'cost_explorer_payer_billings', 'spot_savings_analysis']

cloud_governance/policy/aws/optimize_resources_report.py

@@ -0,0 +1,113 @@
+import json
+
+import boto3
+
+from cloud_governance.common.clouds.aws.support.support_operations import SupportOperations
+from cloud_governance.common.clouds.aws.utils.common_methods import get_tag_value_from_tags
+from cloud_governance.common.logger.init_logger import logger
+
+# @Todo, focusing only on cost-optimizing service: Need to find the tags for, rds, route53, ecr
+
+
+class OptimizeResourcesReport:
+    COST_OPIMIZING_REPORTS = {
+        'ec2_reports': ['Amazon EC2 instances over-provisioned for Microsoft SQL Server',
+                        'Low Utilization Amazon EC2 Instances',
+                        'Amazon EC2 instances consolidation for Microsoft SQL Server',
+                        'Amazon EC2 Reserved Instance Lease Expiration', 'Amazon EC2 Instances Stopped'],
+        'ebs_reports': ['Amazon EBS over-provisioned volumes', 'Underutilized Amazon EBS Volumes'],
+        'eip_reports': ['Unassociated Elastic IP Addresses'],
+        's3_reports': ['Amazon S3 Bucket Lifecycle Policy Configured',
+                       'Amazon S3 version-enabled buckets without lifecycle policies configured'],
+        'rds_reports': ['Amazon RDS Idle DB Instances'],
+        'load_balancer_reports': ['Idle Load Balancers'],
+        'lambda_reports': ['AWS Lambda Functions with Excessive Timeouts', 'AWS Lambda Functions with High Error Rates',
+                           'AWS Lambda over-provisioned functions for memory size'],
+        'ecr_reports': ['Amazon ECR Repository Without Lifecycle Policy Configured'],
+        'route_53_reports': ['Amazon Route 53 Latency Resource Record Sets', 'Amazon Route 53 Name Server Delegations']
+    }
+
+    def __init__(self):
+        self.__support_operations = SupportOperations()
+
+    def __get_tags(self, name: str, region_name: str, resource_id: str):
+        """
+        This method returns the aws client
+        :param name:
+        :return:
+        """
+        ec2_reports = self.COST_OPIMIZING_REPORTS['ec2_reports']
+        ebs_reports = self.COST_OPIMIZING_REPORTS['ebs_reports']
+        eip_reports = self.COST_OPIMIZING_REPORTS['eip_reports']
+        s3_reports = self.COST_OPIMIZING_REPORTS['s3_reports']
+        load_balancer_reports = self.COST_OPIMIZING_REPORTS['load_balancer_reports']
+
+        def lower_data(data: list):
+            return list(map(lambda item: item.lower(), data))
+        try:
+            if name.lower() in lower_data(ec2_reports) + lower_data(ebs_reports) + lower_data(eip_reports):
+                return boto3.client('ec2', region_name=region_name). \
+                    describe_tags(Filters=[{'Name': 'resource-id', 'Values': [resource_id]}]).get('Tags', [])
+            elif name.lower() in lower_data(s3_reports):
+               return boto3.client('s3', region_name=region_name).get_bucket_tagging(Bucket=resource_id).get('TagSet', [])
+            elif name.lower() in lower_data(load_balancer_reports):
+                tags = boto3.client('elb', region_name=region_name).\
+                    describe_tags(LoadBalancerNames=[resource_id]).get('TagDescriptions', [])
+                for tag in tags:
+                    if tag.get('LoadBalancerName') == resource_id:
+                        return tag.get('Tags')
+                tags = boto3.client('elbv2', region_name=region_name). \
+                    describe_tags(ResourceArns=[resource_id]).get('TagDescriptions', [])
+                for tag in tags:
+                    if tag.get('ResourceArn') == resource_id:
+                        return tag.get('Tags')
+        except Exception as err:
+            logger.error(err)
+        return []
+
+    def __get_optimization_reports(self):
+        """
+        This method returns the report data
+        :return: 
+        :rtype: 
+        """
+        report_list = self.__support_operations.get_trusted_advisor_reports()
+        optimize_resource_list = []
+        unique_report = set()
+        for report_name, resources in report_list.items():
+            if report_name:
+                for key, values in resources.items():
+                    name = values.get('metadata', {}).get('name')
+                    unique_report.add(name)
+                    flagged_resources_list = values.get('reports', {}).get('flaggedResources')
+                    if flagged_resources_list:
+                        resource_values = [item.replace(' ', '') for item in
+                                           values.get('metadata', {}).get('metadata', [])]
+                        for flagged_resources in flagged_resources_list:
+                            resource_location = ''
+                            if report_name == 'cost_optimizing':
+                                resource_location = 1
+                            resources = {}
+                            for idx, item in enumerate(flagged_resources.get('metadata', [])):
+                                if resource_values[idx] in [f'Day{i}' for i in range(1, 15)]:
+                                    continue
+                                resources[resource_values[idx]] = str(item)
+                                if resource_location and resource_location == idx:
+                                    tags = self.__get_tags(name, region_name=flagged_resources.get('region'), resource_id=item)
+                                    user = get_tag_value_from_tags(tags=tags, tag_name='User')
+                                    resources['User'] = user
+                                    resources['ResourceId'] = item
+                            resources['ReportName'] = name
+                            resources['Report'] = report_name
+                            if 'LastUpdatedTime' in resources:
+                                del resources['LastUpdatedTime']
+                            optimize_resource_list.append(resources)
+        return optimize_resource_list
+
+    def run(self):
+        """
+        This method start the report collection
+        :return: 
+        :rtype: 
+        """
+        return self.__get_optimization_reports()

cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py

@@ -22,19 +22,23 @@ def run(self):
         for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass):
             if self._policy.replace('_', '') == cls[0].lower():
                 response = cls[1]().run()
-                if isinstance(response, str):
-                    logger.info(f'key: {cls[0]}, Response: {response}')
-                else:
-                    if self._policy != 'skipped_resources':
+                if response:
+                    if isinstance(response, str):
+                        logger.info(f'key: {cls[0]}, Response: {response}')
+                    else:
                         logger.info(f'key: {cls[0]}, count: {len(response)}, {response}')
                         policy_result = response
 
                         if self._es_operations.check_elastic_search_connection():
                             if policy_result:
-                                for policy_dict in policy_result:
-                                    policy_dict['region_name'] = self._region
-                                    policy_dict['account'] = self._account
-                                    self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index)
+                                if len(policy_result) > 500:
+                                    self._es_operations.upload_data_in_bulk(data_items=policy_result.copy(),
+                                                                            index=self._es_index)
+                                else:
+                                    for policy_dict in policy_result:
+                                        policy_dict['region_name'] = self._region
+                                        policy_dict['account'] = self._account
+                                        self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index)
                                 logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}')
                             else:
                                 logger.error(f'No data to upload on @{self._account}  at {datetime.utcnow()}')

jenkins/clouds/aws/daily/policies/run_policies.py

@@ -58,6 +58,7 @@ def get_policies(type: str = None):
 policies.remove('cost_billing_reports')
 policies.remove('cost_explorer_payer_billings')
 policies.remove('spot_savings_analysis')
+policies.remove('optimize_resources_report')
 
 es_index_env_var = f'-e es_index={ES_INDEX}' if ES_INDEX else ''
 
@@ -113,8 +114,17 @@ def get_policies(type: str = None):
     envs = list(map(combine_vars, account.items()))
     os.system(f"""podman run --rm --name cloud-governance --net="host" -e policy="send_aggregated_alerts" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -e DEFAULT_ADMINS="['athiruma']"   quay.io/ebattat/cloud-governance:latest""")
 
-# # Git-leaks run on GitHub not related to any aws account
+
+# Running the trust advisor reports, data dumped into default index - cloud-governance-policy-es-index
+
+os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}"  -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}"  -e es_index=""  -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+
+
+# # Git-leaks run on github not related to any aws account
 os.system("echo Run Git-leaks")
+
 region = 'us-east-1'
 policy = 'gitleaks'
 os.system(f"""podman run --rm --name cloud-governance -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_PERF}" -e AWS_DEFAULT_REGION="{region}" -e git_access_token="{GITHUB_TOKEN}" -e git_repo="https://github.com/redhat-performance" -e several_repos="yes" -e policy_output="s3://{BUCKET_PERF}/{LOGS}/$region" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")

tests/unittest/cloud_governance/common/clouds/aws/support/test_support_operations.py

@@ -0,0 +1,70 @@
+from unittest.mock import patch
+
+from cloud_governance.common.clouds.aws.support.support_operations import SupportOperations
+
+
+@patch('boto3.client')
+def test_get_describe_trusted_advisor_checks(mock_client):
+    """
+    This method tests the get_describe_trusted_advisor_checks method
+    :param mock_client:
+    :type mock_client:
+    :return:
+    :rtype:
+    """
+    mock_client.return_value.describe_trusted_advisor_checks.return_value = {
+        'checks': [
+            {
+                'id': 'test_report',
+                'name': 'Test Report',
+                'category': 'test_optimize_report',
+                'metadata': [
+                    'ResourceId',
+                ]
+            },
+        ]
+    }
+    support_operations = SupportOperations()
+    response = support_operations.get_describe_trusted_advisor_checks()
+    assert response == [{'id': 'test_report', 'name': 'Test Report', 'category': 'test_optimize_report', 'metadata': ['ResourceId']}]
+
+
+@patch('boto3.client')
+def test_get_trusted_advisor_reports(mock_client):
+    """
+    This method tests the get_trusted_advisor_reports method
+    :param mock_client:
+    :type mock_client:
+    :return:
+    :rtype:
+    """
+    mock_client.return_value.describe_trusted_advisor_checks.return_value = {
+        'checks': [
+            {
+                'id': 'test_report',
+                'name': 'Test Report',
+                'category': 'test_optimize_report',
+                'metadata': [
+                    'ResourceId',
+                ]
+            },
+        ]
+    }
+    mock_client.return_value.describe_trusted_advisor_check_result.return_value = {
+        'result': {
+            'checkId': 'test_report',
+            'resourcesSummary': {
+                'resourcesProcessed': 123,
+            },
+            'flaggedResources': [
+                {
+                    'metadata': [
+                        'test-123',
+                    ]
+                },
+            ]
+        }
+    }
+    support_operations = SupportOperations()
+    response = support_operations.get_trusted_advisor_reports()
+    assert response == {'test_optimize_report': {'test_report': {'metadata': {'id': 'test_report', 'name': 'Test Report', 'category': 'test_optimize_report', 'metadata': ['ResourceId']}, 'reports': {'checkId': 'test_report', 'resourcesSummary': {'resourcesProcessed': 123}, 'flaggedResources': [{'metadata': ['test-123']}]}}}}