From ecd4985bcce0e12eca9c1bed19f45680d83acf81 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti <97395760+athiruma@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:02:05 +0530 Subject: [PATCH] Added the optmize reports policy (#695) --- .../clouds/aws/resource_explorer/__init__.py | 0 .../resource_explorer_operations.py | 52 ++++++++ .../common/clouds/aws/support/__init__.py | 0 .../clouds/aws/support/support_operations.py | 44 +++++++ .../common/clouds/aws/utils/common_methods.py | 28 +++++ .../elasticsearch/elasticsearch_operations.py | 12 +- .../main/environment_variables.py | 2 +- .../policy/aws/optimize_resources_report.py | 113 ++++++++++++++++++ .../zombie_non_cluster_polices.py | 20 ++-- .../clouds/aws/daily/policies/run_policies.py | 12 +- .../common/clouds/aws/support/__init__.py | 0 .../aws/support/test_support_operations.py | 70 +++++++++++ .../aws/test_optimize_resources_report.py | 49 ++++++++ 13 files changed, 389 insertions(+), 13 deletions(-) create mode 100644 cloud_governance/common/clouds/aws/resource_explorer/__init__.py create mode 100644 cloud_governance/common/clouds/aws/resource_explorer/resource_explorer_operations.py create mode 100644 cloud_governance/common/clouds/aws/support/__init__.py create mode 100644 cloud_governance/common/clouds/aws/support/support_operations.py create mode 100644 cloud_governance/common/clouds/aws/utils/common_methods.py create mode 100644 cloud_governance/policy/aws/optimize_resources_report.py create mode 100644 tests/unittest/cloud_governance/common/clouds/aws/support/__init__.py create mode 100644 tests/unittest/cloud_governance/common/clouds/aws/support/test_support_operations.py create mode 100644 tests/unittest/cloud_governance/policy/aws/test_optimize_resources_report.py diff --git a/cloud_governance/common/clouds/aws/resource_explorer/__init__.py b/cloud_governance/common/clouds/aws/resource_explorer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/aws/resource_explorer/resource_explorer_operations.py b/cloud_governance/common/clouds/aws/resource_explorer/resource_explorer_operations.py new file mode 100644 index 00000000..7d5ef11d --- /dev/null +++ b/cloud_governance/common/clouds/aws/resource_explorer/resource_explorer_operations.py @@ -0,0 +1,52 @@ +import boto3 + +from cloud_governance.common.logger.init_logger import logger + +# @Todo, This class will be used in the feature releases. +# @Todo, it helps in find the resource data like tags by using search query + + +class ResourceExplorerOperations: + """ + This class performs the resource explorer operations + """ + + def __init__(self): + self.__client = self.__set_client() + + def __set_client(self): + view = self.list_views() + region = 'us-east-1' + if view: + region = view.split(':')[3] + return boto3.client('resource-explorer-2', region_name=region) + + def __search(self, search_string: str): + try: + response = self.__client.search(QueryString=search_string) + return response.get('Resources', []) + except Exception as err: + logger.error(err) + return [] + + def find_resource_tags(self, resource_id: str): + search_results = self.__search(search_string=f'"{resource_id}"') + tags = [] + for resource in search_results: + if resource_id in resource.get('Arn'): + if resource.get('Properties'): + tags = resource.get('Properties', {})[0].get('Data') + return tags + + def list_views(self): + """ + This method returns list the views + :return: + :rtype: + """ + client = boto3.client('resource-explorer-2', region_name='us-east-1') + views = client.list_views()['Views'] + if views: + return views[0] + else: + raise Exception("No Resource Explorer view found in Region: us-east-1, create one on Free of Charge") diff --git a/cloud_governance/common/clouds/aws/support/__init__.py b/cloud_governance/common/clouds/aws/support/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/aws/support/support_operations.py b/cloud_governance/common/clouds/aws/support/support_operations.py new file mode 100644 index 00000000..4859762d --- /dev/null +++ b/cloud_governance/common/clouds/aws/support/support_operations.py @@ -0,0 +1,44 @@ +import boto3 + +from cloud_governance.common.logger.init_logger import logger + + +class SupportOperations: + """ + This class performs the support operations + """ + + def __init__(self): + self.__client = boto3.client('support', region_name='us-east-1') + + def get_describe_trusted_advisor_checks(self): + """ + This method returns the trusted advisor check results + :return: + :rtype: + """ + try: + response = self.__client.describe_trusted_advisor_checks(language='en') + return response.get('checks', []) + except Exception as err: + logger.error(err) + return [] + + def get_trusted_advisor_reports(self): + """ + This method returns the reports of the checks + :return: + :rtype: + """ + result = {} + try: + advisor_checks_list = self.get_describe_trusted_advisor_checks() + for check in advisor_checks_list: + response = self.__client.describe_trusted_advisor_check_result(checkId=check.get('id')) + result.setdefault(check.get('category'), {}).setdefault(check.get('id'), { + 'metadata': check, + 'reports': response.get('result', []) + }) + except Exception as err: + logger.err(err) + return result diff --git a/cloud_governance/common/clouds/aws/utils/common_methods.py b/cloud_governance/common/clouds/aws/utils/common_methods.py new file mode 100644 index 00000000..fbd1c6e6 --- /dev/null +++ b/cloud_governance/common/clouds/aws/utils/common_methods.py @@ -0,0 +1,28 @@ +def get_tag_value_from_tags(tags: list, tag_name: str, cast_type: str = 'str', + default_value: any = '') -> any: + """ + This method returns the tag value inputted by tag_name + :param tags: + :type tags: + :param tag_name: + :type tag_name: + :param cast_type: + :type cast_type: + :param default_value: + :type default_value: + :return: + :rtype: + """ + if tags: + for tag in tags: + key = tag.get('Key').lower().replace("_", '').replace("-", '').strip() + if key == tag_name.lower(): + if cast_type: + if cast_type == 'int': + return int(tag.get('Value').split()[0].strip()) + elif cast_type == 'float': + return float(tag.get('Value').strip()) + else: + return str(tag.get('Value').strip()) + return tag.get('Value').strip() + return default_value diff --git a/cloud_governance/common/elasticsearch/elasticsearch_operations.py b/cloud_governance/common/elasticsearch/elasticsearch_operations.py index 65bcc39f..bf723d5a 100644 --- a/cloud_governance/common/elasticsearch/elasticsearch_operations.py +++ b/cloud_governance/common/elasticsearch/elasticsearch_operations.py @@ -311,10 +311,16 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs): if kwargs.get('id'): item['_id'] = item.get(kwargs.get('id')) if not item.get('timestamp'): - item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d") + if 'CurrentDate' in item: + item['timestamp'] = datetime.strptime(item.get('CurrentDate'), "%Y-%m-%d") + else: + item['timestamp'] = datetime.utcnow() item['_index'] = index - item['AccountId'] = str(item.get('AccountId')) - item['Policy'] = self.__environment_variables_dict.get('policy') + if item.get('AccountId'): + item['AccountId'] = str(item.get('AccountId')) + if 'account' not in item: + item['account'] = self.__account + item['policy'] = self.__environment_variables_dict.get('policy') response = bulk(self.__es, bulk_items) if response: total_uploaded += len(bulk_items) diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index fcbba8f8..9666f3dc 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -53,7 +53,7 @@ def __init__(self): 'empty_roles', 'ip_unattached', 'unused_nat_gateway', 'zombie_snapshots', 'skipped_resources', - 'monthly_report'] + 'monthly_report', 'optimize_resources_report'] es_index = 'cloud-governance-policy-es-index' self._environment_variables_dict['cost_policies'] = ['cost_explorer', 'cost_over_usage', 'cost_billing_reports', 'cost_explorer_payer_billings', 'spot_savings_analysis'] diff --git a/cloud_governance/policy/aws/optimize_resources_report.py b/cloud_governance/policy/aws/optimize_resources_report.py new file mode 100644 index 00000000..b5c0888a --- /dev/null +++ b/cloud_governance/policy/aws/optimize_resources_report.py @@ -0,0 +1,113 @@ +import json + +import boto3 + +from cloud_governance.common.clouds.aws.support.support_operations import SupportOperations +from cloud_governance.common.clouds.aws.utils.common_methods import get_tag_value_from_tags +from cloud_governance.common.logger.init_logger import logger + +# @Todo, focusing only on cost-optimizing service: Need to find the tags for, rds, route53, ecr + + +class OptimizeResourcesReport: + COST_OPIMIZING_REPORTS = { + 'ec2_reports': ['Amazon EC2 instances over-provisioned for Microsoft SQL Server', + 'Low Utilization Amazon EC2 Instances', + 'Amazon EC2 instances consolidation for Microsoft SQL Server', + 'Amazon EC2 Reserved Instance Lease Expiration', 'Amazon EC2 Instances Stopped'], + 'ebs_reports': ['Amazon EBS over-provisioned volumes', 'Underutilized Amazon EBS Volumes'], + 'eip_reports': ['Unassociated Elastic IP Addresses'], + 's3_reports': ['Amazon S3 Bucket Lifecycle Policy Configured', + 'Amazon S3 version-enabled buckets without lifecycle policies configured'], + 'rds_reports': ['Amazon RDS Idle DB Instances'], + 'load_balancer_reports': ['Idle Load Balancers'], + 'lambda_reports': ['AWS Lambda Functions with Excessive Timeouts', 'AWS Lambda Functions with High Error Rates', + 'AWS Lambda over-provisioned functions for memory size'], + 'ecr_reports': ['Amazon ECR Repository Without Lifecycle Policy Configured'], + 'route_53_reports': ['Amazon Route 53 Latency Resource Record Sets', 'Amazon Route 53 Name Server Delegations'] + } + + def __init__(self): + self.__support_operations = SupportOperations() + + def __get_tags(self, name: str, region_name: str, resource_id: str): + """ + This method returns the aws client + :param name: + :return: + """ + ec2_reports = self.COST_OPIMIZING_REPORTS['ec2_reports'] + ebs_reports = self.COST_OPIMIZING_REPORTS['ebs_reports'] + eip_reports = self.COST_OPIMIZING_REPORTS['eip_reports'] + s3_reports = self.COST_OPIMIZING_REPORTS['s3_reports'] + load_balancer_reports = self.COST_OPIMIZING_REPORTS['load_balancer_reports'] + + def lower_data(data: list): + return list(map(lambda item: item.lower(), data)) + try: + if name.lower() in lower_data(ec2_reports) + lower_data(ebs_reports) + lower_data(eip_reports): + return boto3.client('ec2', region_name=region_name). \ + describe_tags(Filters=[{'Name': 'resource-id', 'Values': [resource_id]}]).get('Tags', []) + elif name.lower() in lower_data(s3_reports): + return boto3.client('s3', region_name=region_name).get_bucket_tagging(Bucket=resource_id).get('TagSet', []) + elif name.lower() in lower_data(load_balancer_reports): + tags = boto3.client('elb', region_name=region_name).\ + describe_tags(LoadBalancerNames=[resource_id]).get('TagDescriptions', []) + for tag in tags: + if tag.get('LoadBalancerName') == resource_id: + return tag.get('Tags') + tags = boto3.client('elbv2', region_name=region_name). \ + describe_tags(ResourceArns=[resource_id]).get('TagDescriptions', []) + for tag in tags: + if tag.get('ResourceArn') == resource_id: + return tag.get('Tags') + except Exception as err: + logger.error(err) + return [] + + def __get_optimization_reports(self): + """ + This method returns the report data + :return: + :rtype: + """ + report_list = self.__support_operations.get_trusted_advisor_reports() + optimize_resource_list = [] + unique_report = set() + for report_name, resources in report_list.items(): + if report_name: + for key, values in resources.items(): + name = values.get('metadata', {}).get('name') + unique_report.add(name) + flagged_resources_list = values.get('reports', {}).get('flaggedResources') + if flagged_resources_list: + resource_values = [item.replace(' ', '') for item in + values.get('metadata', {}).get('metadata', [])] + for flagged_resources in flagged_resources_list: + resource_location = '' + if report_name == 'cost_optimizing': + resource_location = 1 + resources = {} + for idx, item in enumerate(flagged_resources.get('metadata', [])): + if resource_values[idx] in [f'Day{i}' for i in range(1, 15)]: + continue + resources[resource_values[idx]] = str(item) + if resource_location and resource_location == idx: + tags = self.__get_tags(name, region_name=flagged_resources.get('region'), resource_id=item) + user = get_tag_value_from_tags(tags=tags, tag_name='User') + resources['User'] = user + resources['ResourceId'] = item + resources['ReportName'] = name + resources['Report'] = report_name + if 'LastUpdatedTime' in resources: + del resources['LastUpdatedTime'] + optimize_resource_list.append(resources) + return optimize_resource_list + + def run(self): + """ + This method start the report collection + :return: + :rtype: + """ + return self.__get_optimization_reports() diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py index 8448a0dc..f03f16a5 100644 --- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py +++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py @@ -22,19 +22,23 @@ def run(self): for cls in inspect.getmembers(zombie_non_cluster_policy_module, inspect.isclass): if self._policy.replace('_', '') == cls[0].lower(): response = cls[1]().run() - if isinstance(response, str): - logger.info(f'key: {cls[0]}, Response: {response}') - else: - if self._policy != 'skipped_resources': + if response: + if isinstance(response, str): + logger.info(f'key: {cls[0]}, Response: {response}') + else: logger.info(f'key: {cls[0]}, count: {len(response)}, {response}') policy_result = response if self._es_operations.check_elastic_search_connection(): if policy_result: - for policy_dict in policy_result: - policy_dict['region_name'] = self._region - policy_dict['account'] = self._account - self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) + if len(policy_result) > 500: + self._es_operations.upload_data_in_bulk(data_items=policy_result.copy(), + index=self._es_index) + else: + for policy_dict in policy_result: + policy_dict['region_name'] = self._region + policy_dict['account'] = self._account + self._es_operations.upload_to_elasticsearch(data=policy_dict.copy(), index=self._es_index) logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}') else: logger.error(f'No data to upload on @{self._account} at {datetime.utcnow()}') diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 761abe6b..36b000fc 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -58,6 +58,7 @@ def get_policies(type: str = None): policies.remove('cost_billing_reports') policies.remove('cost_explorer_payer_billings') policies.remove('spot_savings_analysis') +policies.remove('optimize_resources_report') es_index_env_var = f'-e es_index={ES_INDEX}' if ES_INDEX else '' @@ -113,8 +114,17 @@ def get_policies(type: str = None): envs = list(map(combine_vars, account.items())) os.system(f"""podman run --rm --name cloud-governance --net="host" -e policy="send_aggregated_alerts" -e {' -e '.join(envs)} -e {' -e '.join(common_envs)} -e DEFAULT_ADMINS="['athiruma']" quay.io/ebattat/cloud-governance:latest""") -# # Git-leaks run on GitHub not related to any aws account + +# Running the trust advisor reports, data dumped into default index - cloud-governance-policy-es-index + +os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") +os.system(f"""podman run --rm --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_index="" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") + + +# # Git-leaks run on github not related to any aws account os.system("echo Run Git-leaks") + region = 'us-east-1' policy = 'gitleaks' os.system(f"""podman run --rm --name cloud-governance -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_PERF}" -e AWS_DEFAULT_REGION="{region}" -e git_access_token="{GITHUB_TOKEN}" -e git_repo="https://github.com/redhat-performance" -e several_repos="yes" -e policy_output="s3://{BUCKET_PERF}/{LOGS}/$region" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""") diff --git a/tests/unittest/cloud_governance/common/clouds/aws/support/__init__.py b/tests/unittest/cloud_governance/common/clouds/aws/support/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/common/clouds/aws/support/test_support_operations.py b/tests/unittest/cloud_governance/common/clouds/aws/support/test_support_operations.py new file mode 100644 index 00000000..9aa7b75f --- /dev/null +++ b/tests/unittest/cloud_governance/common/clouds/aws/support/test_support_operations.py @@ -0,0 +1,70 @@ +from unittest.mock import patch + +from cloud_governance.common.clouds.aws.support.support_operations import SupportOperations + + +@patch('boto3.client') +def test_get_describe_trusted_advisor_checks(mock_client): + """ + This method tests the get_describe_trusted_advisor_checks method + :param mock_client: + :type mock_client: + :return: + :rtype: + """ + mock_client.return_value.describe_trusted_advisor_checks.return_value = { + 'checks': [ + { + 'id': 'test_report', + 'name': 'Test Report', + 'category': 'test_optimize_report', + 'metadata': [ + 'ResourceId', + ] + }, + ] + } + support_operations = SupportOperations() + response = support_operations.get_describe_trusted_advisor_checks() + assert response == [{'id': 'test_report', 'name': 'Test Report', 'category': 'test_optimize_report', 'metadata': ['ResourceId']}] + + +@patch('boto3.client') +def test_get_trusted_advisor_reports(mock_client): + """ + This method tests the get_trusted_advisor_reports method + :param mock_client: + :type mock_client: + :return: + :rtype: + """ + mock_client.return_value.describe_trusted_advisor_checks.return_value = { + 'checks': [ + { + 'id': 'test_report', + 'name': 'Test Report', + 'category': 'test_optimize_report', + 'metadata': [ + 'ResourceId', + ] + }, + ] + } + mock_client.return_value.describe_trusted_advisor_check_result.return_value = { + 'result': { + 'checkId': 'test_report', + 'resourcesSummary': { + 'resourcesProcessed': 123, + }, + 'flaggedResources': [ + { + 'metadata': [ + 'test-123', + ] + }, + ] + } + } + support_operations = SupportOperations() + response = support_operations.get_trusted_advisor_reports() + assert response == {'test_optimize_report': {'test_report': {'metadata': {'id': 'test_report', 'name': 'Test Report', 'category': 'test_optimize_report', 'metadata': ['ResourceId']}, 'reports': {'checkId': 'test_report', 'resourcesSummary': {'resourcesProcessed': 123}, 'flaggedResources': [{'metadata': ['test-123']}]}}}} diff --git a/tests/unittest/cloud_governance/policy/aws/test_optimize_resources_report.py b/tests/unittest/cloud_governance/policy/aws/test_optimize_resources_report.py new file mode 100644 index 00000000..874b3325 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/test_optimize_resources_report.py @@ -0,0 +1,49 @@ +from unittest.mock import patch + +from cloud_governance.policy.aws.optimize_resources_report import OptimizeResourcesReport + + +@patch('boto3.client') +def test_get_optimization_reports(mock_client): + """ + This method tests the methods returns the data + :return: + :rtype: + """ + optimize_reports = OptimizeResourcesReport() + mock_client.return_value.describe_trusted_advisor_checks.return_value = { + 'checks': [ + { + 'id': 'test_report', + 'name': 'Test Report', + 'category': 'test_optimize_report', + 'metadata': [ + 'ResourceId', + ] + }, + ] + } + mock_client.return_value.describe_trusted_advisor_check_result.return_value = { + 'result': { + 'checkId': 'test_report', + 'resourcesSummary': { + 'resourcesProcessed': 123, + 'resourcesFlagged': 123, + 'resourcesIgnored': 123, + 'resourcesSuppressed': 123 + }, + 'flaggedResources': [ + { + 'status': 'string', + 'region': 'string', + 'resourceId': 'string', + 'isSuppressed': True, + 'metadata': [ + 'test-123', + ] + }, + ] + } + } + response = optimize_reports.run() + assert response == [{'ResourceId': 'test-123', 'ReportName': 'Test Report', 'Report': 'test_optimize_report'}]