From 5be2c6742a1e2170177fa370c8791425542f19a9 Mon Sep 17 00:00:00 2001 From: Dmytro Afanasiev Date: Thu, 12 Dec 2024 15:46:02 +0200 Subject: [PATCH] Add finops --- src/handlers/high_level_reports_handler.py | 39 +++++++++-- src/helpers/constants.py | 4 ++ .../processors/new_metrics_collector.py | 51 +++++++++++++- src/services/metadata.py | 21 ++++++ src/services/reports.py | 32 +++++++++ src/validators/swagger_request_models.py | 1 + tests/conftest.py | 16 ++++- .../metrics/aws_operational_finops.json | 37 ++++++++++ .../expected/operational/finops_report.json | 47 ++++++++++--- tests/data/metadata/finops.json | 19 +++++ tests/test_endpoints/conftest.py | 69 ++++++++++++++++++- .../test_operational_report.py | 36 ++++++++++ .../test_metrics/test_pipeline.py | 8 ++- 13 files changed, 361 insertions(+), 19 deletions(-) create mode 100644 tests/data/expected/metrics/aws_operational_finops.json create mode 100644 tests/data/metadata/finops.json diff --git a/src/handlers/high_level_reports_handler.py b/src/handlers/high_level_reports_handler.py index ab6a7186..a847b116 100644 --- a/src/handlers/high_level_reports_handler.py +++ b/src/handlers/high_level_reports_handler.py @@ -34,6 +34,7 @@ ReportType.OPERATIONAL_RESOURCES: 'CUSTODIAN_RESOURCES_REPORT', ReportType.OPERATIONAL_OVERVIEW: 'CUSTODIAN_OVERVIEW_REPORT', ReportType.OPERATIONAL_RULES: 'CUSTODIAN_RULES_REPORT', + ReportType.OPERATIONAL_FINOPS: 'CUSTODIAN_FINOPS_REPORT', # ReportType.OPERATIONAL_COMPLIANCE: 'CUSTODIAN_COMPLIANCE_REPORT', ReportType.C_LEVEL_OVERVIEW: 'CUSTODIAN_CUSTOMER_OVERVIEW_REPORT', } @@ -66,6 +67,8 @@ def convert_to_old_rt(rt: ReportType) -> str: return 'COMPLIANCE' case ReportType.OPERATIONAL_RULES: return 'RULE' + case ReportType.OPERATIONAL_FINOPS: + return 'FINOPS' def __init__(self, receivers: tuple[str, ...] = (), size_limit: int = 0): self._receivers = receivers # base receivers @@ -115,7 +118,7 @@ def _operational_resources_custom(rep: ReportMetrics) -> dict: } @staticmethod - def _operational_rules(rep: ReportMetrics) -> dict: + def _operational_rules_custom(rep: ReportMetrics) -> dict: assert rep.type == ReportType.OPERATIONAL_RULES data = rep.data.as_dict() return { @@ -130,6 +133,29 @@ def _operational_rules(rep: ReportMetrics) -> dict: }, } + @staticmethod + def _operational_finops_custom(rep: ReportMetrics) -> dict: + assert rep.type == ReportType.OPERATIONAL_FINOPS + data = rep.data.as_dict() + result = [] + for ss, rules in data['data'].items(): + for rule in rules: + rule['region_data'] = { + region: {'resources': resources} + for region, resources in rule.pop('resources', {}).items() + } + + result.append({'service_section': ss, 'rules_data': rules}) + + return { + 'tenant_name': rep.tenant, + 'id': data['id'], + 'cloud': rep.cloud.value, # pyright: ignore + 'activated_regions': data['activated_regions'], + 'last_scan_date': data['last_scan_date'], + 'data': result, + } + def build_base(self, rep: ReportMetrics) -> dict: return { 'receivers': self._receivers, @@ -154,7 +180,9 @@ def convert(self, rep: ReportMetrics) -> dict | None: case ReportType.OPERATIONAL_RESOURCES: custom = self._operational_resources_custom(rep) case ReportType.OPERATIONAL_RULES: - custom = self._operational_rules(rep) + custom = self._operational_rules_custom(rep) + case ReportType.OPERATIONAL_FINOPS: + custom = self._operational_finops_custom(rep) case _: return base.update(custom) @@ -225,8 +253,11 @@ def post_c_level(self, event: CLevelGetReportModel): previous_data = previous.data.as_dict() current_data = rep.data.as_dict() for cl, data in current_data.items(): - add_diff(data, previous_data.get(cl, {}), - exclude=('total_scanned_tenants', )) + add_diff( + data, + previous_data.get(cl, {}), + exclude=('total_scanned_tenants',), + ) base = builder.build_base(rep) base['data'] = current_data diff --git a/src/helpers/constants.py b/src/helpers/constants.py index 5565dd13..5c4fda6f 100644 --- a/src/helpers/constants.py +++ b/src/helpers/constants.py @@ -1041,6 +1041,10 @@ def start(self, now: datetime) -> datetime | None: 'OPERATIONAL_COMPLIANCE', 'Compliance per tenant as of date of generation', ) + OPERATIONAL_FINOPS = ( + 'OPERATIONAL_FINOPS', + 'Finops report per tenant as of date of generation', + ) # C-Level, kind of for the whole customer C_LEVEL_OVERVIEW = ( diff --git a/src/lambdas/custodian_metrics_updater/processors/new_metrics_collector.py b/src/lambdas/custodian_metrics_updater/processors/new_metrics_collector.py index 676550f8..7565bac6 100644 --- a/src/lambdas/custodian_metrics_updater/processors/new_metrics_collector.py +++ b/src/lambdas/custodian_metrics_updater/processors/new_metrics_collector.py @@ -247,6 +247,7 @@ def collect_metrics_for_customer(self, ctx: MetricsContext): ReportType.OPERATIONAL_OVERVIEW, ReportType.OPERATIONAL_RESOURCES, ReportType.OPERATIONAL_RULES, + ReportType.OPERATIONAL_FINOPS, ReportType.C_LEVEL_OVERVIEW, ) @@ -301,6 +302,15 @@ def collect_metrics_for_customer(self, ctx: MetricsContext): ) ) ) + _LOG.info('Generating operational finops for all tenants') + ctx.add_reports( + self.operational_finops( + ctx=ctx, + job_source=job_source, + sc_provider=sc_provider, + report_type=ReportType.OPERATIONAL_FINOPS, + ) + ) # todo operational compliance, attacks, finops, kubernetes # todo project reports @@ -455,6 +465,43 @@ def operational_rules( start=start, ) + def operational_finops( + self, + ctx: MetricsContext, + job_source: JobMetricsDataSource, + sc_provider: ShardsCollectionProvider, + report_type: ReportType, + ) -> ReportsGen: + start = report_type.start(ctx.now) + end = report_type.end(ctx.now) + js = job_source.subset(start=start, end=end) + for tenant_name in js.scanned_tenants: + tenant = self._get_tenant(tenant_name) + if not tenant: + _LOG.warning(f'Tenant with name {tenant_name} not found!') + continue + col = sc_provider.get_for_tenant(tenant, end) + if col is None: + _LOG.warning( + f'Cannot get shards collection for ' + f'{tenant.name} for {end}' + ) + continue + data = { + 'id': tenant.project, + 'data': ShardsCollectionDataSource(col, ctx.metadata).finops(), + 'last_scan_date': js.subset(tenant=tenant.name).last_scan_date, + 'activated_regions': sorted( + modular_helpers.get_tenant_regions(tenant) + ), + } + yield self._rms.create( + key=self._rms.key_for_tenant(report_type, tenant), + data=data, # TODO: test whether it's ok to assign large objects to PynamoDB's MapAttribute + end=end, + start=start, + ) + def c_level_overview( self, ctx: MetricsContext, @@ -573,9 +620,7 @@ def _cloud_licenses_info( ) else: cloud_rulesets.append( - RulesetName( - rs.name, rs.version - ).to_str() + RulesetName(rs.name, rs.version).to_str() ) if not cloud_licenses: return {'activated': False, 'license_properties': {}} diff --git a/src/services/metadata.py b/src/services/metadata.py index 09154b50..cb5a6df4 100644 --- a/src/services/metadata.py +++ b/src/services/metadata.py @@ -53,6 +53,14 @@ def __repr__(self) -> str: f'{__name__}.{self.__class__.__name__} object at {hex(id(self))}' ) + def is_finops(self) -> bool: + return 'finops' in self.category.lower() + + def finops_category(self) -> str | None: + if not self.is_finops(): + return + return self.category.split('>')[-1].strip() + class DomainMetadata( msgspec.Struct, kw_only=True, array_like=True, frozen=True, eq=False @@ -215,6 +223,19 @@ def get_no_cache( ) return self._dec.decode(gzip.decompress(data)) + def set( + self, + metadata: Metadata, + lic: 'License', + version: Version = DEFAULT_VERSION, + ): + self._s3.put_object( + bucket=self._env.default_reports_bucket_name(), + key=ReportMetaBucketsKeys.meta_key(lic.license_key, version), + body=gzip.compress(msgspec.msgpack.encode(metadata)), + content_encoding='gzip', + ) + def get( self, lic: 'License', /, *, version: Version = DEFAULT_VERSION ) -> Metadata: diff --git a/src/services/reports.py b/src/services/reports.py index 875357b2..9b6627dd 100644 --- a/src/services/reports.py +++ b/src/services/reports.py @@ -406,6 +406,38 @@ def resource_types(self) -> dict[str, int]: result[rt] += len(res) return result + def finops(self) -> dict[str, list[dict]]: + """ + Produces finops data in its old format + """ + res = {} + for rule in self._resources: + rule_meta = self._meta.rule(rule) + finops_category = rule_meta.finops_category() + if not finops_category: + continue # not a finops rule + ss = rule_meta.service_section + if not ss: + _LOG.warning(f'Rule {rule} does not have service section') + continue + res.setdefault(ss, []).append( + { + 'rule': self._col.meta[rule].get('description', rule), + 'service': rule_meta.service + or service_from_resource_type( + self._col.meta[rule]['resource'] + ), + 'category': finops_category, + 'severity': rule_meta.severity.value, + 'resource_type': self._col.meta[rule]['resource'], + 'resources': { + region: list(res) + for region, res in self._resources[rule].items() + }, + } + ) + return res + class ShardsCollectionProvider: """ diff --git a/src/validators/swagger_request_models.py b/src/validators/swagger_request_models.py index 3390f889..2624911b 100644 --- a/src/validators/swagger_request_models.py +++ b/src/validators/swagger_request_models.py @@ -1152,6 +1152,7 @@ def new_types(self) -> tuple[ReportType, ...]: 'RESOURCES': ReportType.OPERATIONAL_RESOURCES, # 'COMPLIANCE': ReportType.OPERATIONAL_COMPLIANCE, 'RULE': ReportType.OPERATIONAL_RULES, + 'FINOPS': ReportType.OPERATIONAL_FINOPS } if not self.types: return tuple(old_new.values()) diff --git a/tests/conftest.py b/tests/conftest.py index 89133614..e832ed0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os from datetime import datetime, timedelta from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable from unittest.mock import patch import mongomock @@ -240,3 +240,17 @@ def empty_metadata() -> 'Metadata': from services.metadata import Metadata return Metadata.empty() + + +@pytest.fixture +def load_metadata() -> Callable[[str], 'Metadata']: + from services.metadata import Metadata + + def _inner(name: str) -> Metadata: + if not name.endswith('.json'): + name = f'{name}.json' + path = DATA / 'metadata' / name + assert path.exists(), f'{path} must exist' + with open(path, 'rb') as fp: + return msgspec.json.decode(fp.read(), type=Metadata) + return _inner diff --git a/tests/data/expected/metrics/aws_operational_finops.json b/tests/data/expected/metrics/aws_operational_finops.json new file mode 100644 index 00000000..a79da15d --- /dev/null +++ b/tests/data/expected/metrics/aws_operational_finops.json @@ -0,0 +1,37 @@ +{ + "id": "123456789012", + "data": { + "Compute": [ + { + "rule": "Description for ecc-aws-070-unused_ec2_security_groups", + "service": "Security Group", + "category": "Unused Resources", + "severity": "Unknown", + "resource_type": "aws.security-group", + "resources": { + "eu-central-1": [ + { + "id": "sg-05209ceeb761317e5", + "name": "west-eu" + }, + { + "id": "sg-09d19538187df66cf", + "name": "asia" + }, + { + "id": "sg-0ae266faddaef17e9", + "name": "packer-ssh-access" + } + ] + } + } + ] + }, + "last_scan_date": "2024-12-12T00:38:12.463910Z", + "activated_regions": [ + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-3" + ] +} \ No newline at end of file diff --git a/tests/data/expected/operational/finops_report.json b/tests/data/expected/operational/finops_report.json index 629114c9..c0d4e1d8 100644 --- a/tests/data/expected/operational/finops_report.json +++ b/tests/data/expected/operational/finops_report.json @@ -2,21 +2,52 @@ "receivers": [ "admin@gmail.com" ], + "report_type": "FINOPS", "customer": "TEST_CUSTOMER", + "from": "2024-12-05T13:42:56.472339Z", + "to": "2024-12-12T13:42:56.472339Z", + "outdated_tenants": [], + "externalData": false, + "data": [ + { + "service_section": "Compute", + "rules_data": [ + { + "rule": "Description for ecc-aws-070-unused_ec2_security_groups", + "service": "Security Group", + "category": "Unused Resources", + "severity": "Unknown", + "resource_type": "aws.security-group", + "region_data": { + "eu-central-1": { + "resources": [ + { + "id": "sg-05209ceeb761317e5", + "name": "west-eu" + }, + { + "id": "sg-09d19538187df66cf", + "name": "asia" + }, + { + "id": "sg-0ae266faddaef17e9", + "name": "packer-ssh-access" + } + ] + } + } + } + ] + } + ], "tenant_name": "AWS-TESTING", "id": "123456789012", - "cloud": "aws", + "cloud": "AWS", "activated_regions": [ "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-3" ], - "from": "2024-11-03T00:00:00Z", - "to": "2024-11-10T00:00:00Z", - "last_scan_date": "2024-11-04T00:00:00Z", - "outdated_tenants": [], - "data": [], - "externalData": false, - "report_type": "FINOPS" + "last_scan_date": "2024-12-12T00:38:12.463910Z" } \ No newline at end of file diff --git a/tests/data/metadata/finops.json b/tests/data/metadata/finops.json new file mode 100644 index 00000000..c4594973 --- /dev/null +++ b/tests/data/metadata/finops.json @@ -0,0 +1,19 @@ +{ + "rules": { + "ecc-aws-070-unused_ec2_security_groups": [ + "testing", + "FinOps > Unused Resources", + "Compute", + "Security Group", + "Unknown", + [], + true, + "", + "", + "", + {}, + {}, + {} + ] + } +} \ No newline at end of file diff --git a/tests/test_endpoints/conftest.py b/tests/test_endpoints/conftest.py index 7762f248..9987a427 100644 --- a/tests/test_endpoints/conftest.py +++ b/tests/test_endpoints/conftest.py @@ -1,11 +1,10 @@ import json import uuid from datetime import timedelta, datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable import boto3 import pytest -from dateutil.relativedelta import relativedelta, SU from moto.backends import get_backend from webtest import TestApp @@ -17,6 +16,7 @@ if TYPE_CHECKING: from modular_sdk.models.tenant import Tenant from modular_sdk.models.customer import Customer + from services.license_service import License # assuming that only this package will use mongo so that we need to clear @@ -217,3 +217,68 @@ def factory(tenant, submitted_at, ) return factory + + +@pytest.fixture() +def main_license(main_customer) -> 'License': + lic = SP.license_service.create( + license_key='license-key', + customer=main_customer.name, + created_by='testing', + customers={main_customer.name: {'attachment_model': 'permitted', 'tenants': [], 'tenant_license_key': 'tlk'}}, + description='Testing license', + expiration=utc_iso(utc_datetime() + timedelta(days=1)), + ruleset_ids=['AWS', 'AZURE', 'GOOGLE'], + allowance={ + 'balance_exhaustion_model': 'independent', + 'job_balance': 10, + 'time_range': 'DAY' + }, + event_driven={ + 'active': False, + 'quota': 0, + } + ) + SP.license_service.save(lic) + SP.ruleset_service.create( + customer=CAASEnv.SYSTEM_CUSTOMER_NAME.get(), + name='AWS', + version='', + cloud='AWS', + rules=[], + event_driven=False, + licensed=True, + license_keys=[lic.license_key], + license_manager_id='AWS' + ).save() + SP.ruleset_service.create( + customer=CAASEnv.SYSTEM_CUSTOMER_NAME.get(), + name='AZURE', + version='', + cloud='AZURE', + rules=[], + event_driven=False, + licensed=True, + license_keys=[lic.license_key], + license_manager_id='AZURE' + ).save() + SP.ruleset_service.create( + customer=CAASEnv.SYSTEM_CUSTOMER_NAME.get(), + name='GOOGLE', + version='', + cloud='GCP', + rules=[], + event_driven=False, + licensed=True, + license_keys=[lic.license_key], + license_manager_id='GOOGLE' + ).save() + return lic + + +@pytest.fixture() +def set_license_metadata(main_license, load_metadata) -> Callable[[str], None]: + def _inner(name: str): + metadata = load_metadata(name) + SP.metadata_provider.set(metadata, main_license) + return _inner diff --git a/tests/test_endpoints/test_maestro_reports/test_operational_report.py b/tests/test_endpoints/test_maestro_reports/test_operational_report.py index 0078a9f6..db280232 100644 --- a/tests/test_endpoints/test_maestro_reports/test_operational_report.py +++ b/tests/test_endpoints/test_maestro_reports/test_operational_report.py @@ -34,6 +34,15 @@ def aws_operational_rules_metrics(aws_tenant, load_expected, utcnow): ).save() +@pytest.fixture() +def aws_operational_finops_metrics(aws_tenant, load_expected, utcnow): + SP.report_metrics_service.create( + key=SP.report_metrics_service.key_for_tenant(ReportType.OPERATIONAL_FINOPS, aws_tenant), + data=load_expected('metrics/aws_operational_finops'), + end=utcnow + ).save() + + def validate_maestro_model(m: dict): assert isinstance(m, dict) assert m['viewType'] == 'm3' @@ -134,3 +143,30 @@ def test_operational_rules_report_aws_tenant( json.loads(params[0]['model']['notificationAsJson']), load_expected('operational/rules_report') ) + + +def test_operational_finops_report_aws_tenant( + system_user_token, sre_client, aws_operational_finops_metrics, + mocked_rabbitmq, load_expected +): + resp = sre_client.request( + "/reports/operational", + "POST", + auth=system_user_token, + data={ + "customer_id": "TEST_CUSTOMER", + "tenant_names": ['AWS-TESTING'], + "types": ["FINOPS"], + "receivers": ["admin@gmail.com"] + } + ) + assert resp.status_int == 202 + assert len(mocked_rabbitmq.send_sync.mock_calls) == 1 + params = mocked_rabbitmq.send_sync.mock_calls[0].kwargs['parameters'] + + assert len(params) == 1, 'Only one operational report is sent' + assert params[0]['model']['notificationType'] == 'CUSTODIAN_FINOPS_REPORT' + assert dicts_equal( + json.loads(params[0]['model']['notificationAsJson']), + load_expected('operational/finops_report') + ) diff --git a/tests/test_endpoints/test_metrics/test_pipeline.py b/tests/test_endpoints/test_metrics/test_pipeline.py index 3b395069..f6fed4cd 100644 --- a/tests/test_endpoints/test_metrics/test_pipeline.py +++ b/tests/test_endpoints/test_metrics/test_pipeline.py @@ -200,8 +200,10 @@ def test_metrics_update( aws_tenant, azure_tenant, google_tenant, - main_customer + main_customer, + set_license_metadata ): + set_license_metadata('finops') # todo mock date because currently these tests may fail if executed # in some corner dates resp = sre_client.request('/metrics/update', 'POST', @@ -244,6 +246,10 @@ def test_metrics_update( SP.report_metrics_service.fetch_data_from_s3(item) assert dicts_equal(item.data.as_dict(), load_expected('metrics/google_operational_rules')) + item = SP.report_metrics_service.get_latest_for_tenant(aws_tenant, ReportType.OPERATIONAL_FINOPS) + SP.report_metrics_service.fetch_data_from_s3(item) + assert dicts_equal(item.data.as_dict(), load_expected('metrics/aws_operational_finops')) + def test_metrics_update_c_level( sre_client,