Add finops

src/handlers/high_level_reports_handler.py

@@ -34,6 +34,7 @@
     ReportType.OPERATIONAL_RESOURCES: 'CUSTODIAN_RESOURCES_REPORT',
     ReportType.OPERATIONAL_OVERVIEW: 'CUSTODIAN_OVERVIEW_REPORT',
     ReportType.OPERATIONAL_RULES: 'CUSTODIAN_RULES_REPORT',
+    ReportType.OPERATIONAL_FINOPS: 'CUSTODIAN_FINOPS_REPORT',
     # ReportType.OPERATIONAL_COMPLIANCE: 'CUSTODIAN_COMPLIANCE_REPORT',
     ReportType.C_LEVEL_OVERVIEW: 'CUSTODIAN_CUSTOMER_OVERVIEW_REPORT',
 }
@@ -66,6 +67,8 @@ def convert_to_old_rt(rt: ReportType) -> str:
                 return 'COMPLIANCE'
             case ReportType.OPERATIONAL_RULES:
                 return 'RULE'
+            case ReportType.OPERATIONAL_FINOPS:
+                return 'FINOPS'
 
     def __init__(self, receivers: tuple[str, ...] = (), size_limit: int = 0):
         self._receivers = receivers  # base receivers
@@ -115,7 +118,7 @@ def _operational_resources_custom(rep: ReportMetrics) -> dict:
         }
 
     @staticmethod
-    def _operational_rules(rep: ReportMetrics) -> dict:
+    def _operational_rules_custom(rep: ReportMetrics) -> dict:
         assert rep.type == ReportType.OPERATIONAL_RULES
         data = rep.data.as_dict()
         return {
@@ -130,6 +133,29 @@ def _operational_rules(rep: ReportMetrics) -> dict:
             },
         }
 
+    @staticmethod
+    def _operational_finops_custom(rep: ReportMetrics) -> dict:
+        assert rep.type == ReportType.OPERATIONAL_FINOPS
+        data = rep.data.as_dict()
+        result = []
+        for ss, rules in data['data'].items():
+            for rule in rules:
+                rule['region_data'] = {
+                    region: {'resources': resources}
+                    for region, resources in rule.pop('resources', {}).items()
+                }
+
+            result.append({'service_section': ss, 'rules_data': rules})
+
+        return {
+            'tenant_name': rep.tenant,
+            'id': data['id'],
+            'cloud': rep.cloud.value,  # pyright: ignore
+            'activated_regions': data['activated_regions'],
+            'last_scan_date': data['last_scan_date'],
+            'data': result,
+        }
+
     def build_base(self, rep: ReportMetrics) -> dict:
         return {
             'receivers': self._receivers,
@@ -154,7 +180,9 @@ def convert(self, rep: ReportMetrics) -> dict | None:
             case ReportType.OPERATIONAL_RESOURCES:
                 custom = self._operational_resources_custom(rep)
             case ReportType.OPERATIONAL_RULES:
-                custom = self._operational_rules(rep)
+                custom = self._operational_rules_custom(rep)
+            case ReportType.OPERATIONAL_FINOPS:
+                custom = self._operational_finops_custom(rep)
             case _:
                 return
         base.update(custom)
@@ -225,8 +253,11 @@ def post_c_level(self, event: CLevelGetReportModel):
                 previous_data = previous.data.as_dict()
             current_data = rep.data.as_dict()
             for cl, data in current_data.items():
-                add_diff(data, previous_data.get(cl, {}),
-                         exclude=('total_scanned_tenants', ))
+                add_diff(
+                    data,
+                    previous_data.get(cl, {}),
+                    exclude=('total_scanned_tenants',),
+                )
             base = builder.build_base(rep)
             base['data'] = current_data
 

src/helpers/constants.py

@@ -1041,6 +1041,10 @@ def start(self, now: datetime) -> datetime | None:
         'OPERATIONAL_COMPLIANCE',
         'Compliance per tenant as of date of generation',
     )
+    OPERATIONAL_FINOPS = (
+        'OPERATIONAL_FINOPS',
+        'Finops report per tenant as of date of generation',
+    )
 
     # C-Level, kind of for the whole customer
     C_LEVEL_OVERVIEW = (

src/lambdas/custodian_metrics_updater/processors/new_metrics_collector.py

@@ -247,6 +247,7 @@ def collect_metrics_for_customer(self, ctx: MetricsContext):
             ReportType.OPERATIONAL_OVERVIEW,
             ReportType.OPERATIONAL_RESOURCES,
             ReportType.OPERATIONAL_RULES,
+            ReportType.OPERATIONAL_FINOPS,
             ReportType.C_LEVEL_OVERVIEW,
         )
 
@@ -301,6 +302,15 @@ def collect_metrics_for_customer(self, ctx: MetricsContext):
                 )
             )
         )
+        _LOG.info('Generating operational finops for all tenants')
+        ctx.add_reports(
+            self.operational_finops(
+                ctx=ctx,
+                job_source=job_source,
+                sc_provider=sc_provider,
+                report_type=ReportType.OPERATIONAL_FINOPS,
+            )
+        )
 
         # todo operational compliance, attacks, finops, kubernetes
         # todo project reports
@@ -455,6 +465,43 @@ def operational_rules(
                 start=start,
             )
 
+    def operational_finops(
+        self,
+        ctx: MetricsContext,
+        job_source: JobMetricsDataSource,
+        sc_provider: ShardsCollectionProvider,
+        report_type: ReportType,
+    ) -> ReportsGen:
+        start = report_type.start(ctx.now)
+        end = report_type.end(ctx.now)
+        js = job_source.subset(start=start, end=end)
+        for tenant_name in js.scanned_tenants:
+            tenant = self._get_tenant(tenant_name)
+            if not tenant:
+                _LOG.warning(f'Tenant with name {tenant_name} not found!')
+                continue
+            col = sc_provider.get_for_tenant(tenant, end)
+            if col is None:
+                _LOG.warning(
+                    f'Cannot get shards collection for '
+                    f'{tenant.name} for {end}'
+                )
+                continue
+            data = {
+                'id': tenant.project,
+                'data': ShardsCollectionDataSource(col, ctx.metadata).finops(),
+                'last_scan_date': js.subset(tenant=tenant.name).last_scan_date,
+                'activated_regions': sorted(
+                    modular_helpers.get_tenant_regions(tenant)
+                ),
+            }
+            yield self._rms.create(
+                key=self._rms.key_for_tenant(report_type, tenant),
+                data=data,  # TODO: test whether it's ok to assign large objects to PynamoDB's MapAttribute
+                end=end,
+                start=start,
+            )
+
     def c_level_overview(
         self,
         ctx: MetricsContext,
@@ -573,9 +620,7 @@ def _cloud_licenses_info(
                         )
                     else:
                         cloud_rulesets.append(
-                            RulesetName(
-                                rs.name, rs.version
-                            ).to_str()
+                            RulesetName(rs.name, rs.version).to_str()
                         )
         if not cloud_licenses:
             return {'activated': False, 'license_properties': {}}

src/services/metadata.py

@@ -53,6 +53,14 @@ def __repr__(self) -> str:
             f'{__name__}.{self.__class__.__name__} object at {hex(id(self))}'
         )
 
+    def is_finops(self) -> bool:
+        return 'finops' in self.category.lower()
+
+    def finops_category(self) -> str | None:
+        if not self.is_finops():
+            return
+        return self.category.split('>')[-1].strip()
+
 
 class DomainMetadata(
     msgspec.Struct, kw_only=True, array_like=True, frozen=True, eq=False
@@ -215,6 +223,19 @@ def get_no_cache(
         )
         return self._dec.decode(gzip.decompress(data))
 
+    def set(
+        self,
+        metadata: Metadata,
+        lic: 'License',
+        version: Version = DEFAULT_VERSION,
+    ):
+        self._s3.put_object(
+            bucket=self._env.default_reports_bucket_name(),
+            key=ReportMetaBucketsKeys.meta_key(lic.license_key, version),
+            body=gzip.compress(msgspec.msgpack.encode(metadata)),
+            content_encoding='gzip',
+        )
+
     def get(
         self, lic: 'License', /, *, version: Version = DEFAULT_VERSION
     ) -> Metadata:

src/services/reports.py

@@ -406,6 +406,38 @@ def resource_types(self) -> dict[str, int]:
                 result[rt] += len(res)
         return result
 
+    def finops(self) -> dict[str, list[dict]]:
+        """
+        Produces finops data in its old format
+        """
+        res = {}
+        for rule in self._resources:
+            rule_meta = self._meta.rule(rule)
+            finops_category = rule_meta.finops_category()
+            if not finops_category:
+                continue  # not a finops rule
+            ss = rule_meta.service_section
+            if not ss:
+                _LOG.warning(f'Rule {rule} does not have service section')
+                continue
+            res.setdefault(ss, []).append(
+                {
+                    'rule': self._col.meta[rule].get('description', rule),
+                    'service': rule_meta.service
+                    or service_from_resource_type(
+                        self._col.meta[rule]['resource']
+                    ),
+                    'category': finops_category,
+                    'severity': rule_meta.severity.value,
+                    'resource_type': self._col.meta[rule]['resource'],
+                    'resources': {
+                        region: list(res)
+                        for region, res in self._resources[rule].items()
+                    },
+                }
+            )
+        return res
+
 
 class ShardsCollectionProvider:
     """

src/validators/swagger_request_models.py

@@ -1152,6 +1152,7 @@ def new_types(self) -> tuple[ReportType, ...]:
             'RESOURCES': ReportType.OPERATIONAL_RESOURCES,
             # 'COMPLIANCE': ReportType.OPERATIONAL_COMPLIANCE,
             'RULE': ReportType.OPERATIONAL_RULES,
+            'FINOPS': ReportType.OPERATIONAL_FINOPS
         }
         if not self.types:
             return tuple(old_new.values())

tests/conftest.py

@@ -1,7 +1,7 @@
 import os
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable
 from unittest.mock import patch
 
 import mongomock
@@ -240,3 +240,17 @@ def empty_metadata() -> 'Metadata':
     from services.metadata import Metadata
 
     return Metadata.empty()
+
+
+@pytest.fixture
+def load_metadata() -> Callable[[str], 'Metadata']:
+    from services.metadata import Metadata
+
+    def _inner(name: str) -> Metadata:
+        if not name.endswith('.json'):
+            name = f'{name}.json'
+        path = DATA / 'metadata' / name
+        assert path.exists(), f'{path} must exist'
+        with open(path, 'rb') as fp:
+            return msgspec.json.decode(fp.read(), type=Metadata)
+    return _inner

tests/data/expected/metrics/aws_operational_finops.json

@@ -0,0 +1,37 @@
+{
+  "id": "123456789012",
+  "data": {
+    "Compute": [
+      {
+        "rule": "Description for ecc-aws-070-unused_ec2_security_groups",
+        "service": "Security Group",
+        "category": "Unused Resources",
+        "severity": "Unknown",
+        "resource_type": "aws.security-group",
+        "resources": {
+          "eu-central-1": [
+            {
+              "id": "sg-05209ceeb761317e5",
+              "name": "west-eu"
+            },
+            {
+              "id": "sg-09d19538187df66cf",
+              "name": "asia"
+            },
+            {
+              "id": "sg-0ae266faddaef17e9",
+              "name": "packer-ssh-access"
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "last_scan_date": "2024-12-12T00:38:12.463910Z",
+  "activated_regions": [
+    "eu-central-1",
+    "eu-north-1",
+    "eu-west-1",
+    "eu-west-3"
+  ]
+}

tests/data/expected/operational/finops_report.json

@@ -2,21 +2,52 @@
   "receivers": [
     "[email protected]"
   ],
+  "report_type": "FINOPS",
   "customer": "TEST_CUSTOMER",
+  "from": "2024-12-05T13:42:56.472339Z",
+  "to": "2024-12-12T13:42:56.472339Z",
+  "outdated_tenants": [],
+  "externalData": false,
+  "data": [
+    {
+      "service_section": "Compute",
+      "rules_data": [
+        {
+          "rule": "Description for ecc-aws-070-unused_ec2_security_groups",
+          "service": "Security Group",
+          "category": "Unused Resources",
+          "severity": "Unknown",
+          "resource_type": "aws.security-group",
+          "region_data": {
+            "eu-central-1": {
+              "resources": [
+                {
+                  "id": "sg-05209ceeb761317e5",
+                  "name": "west-eu"
+                },
+                {
+                  "id": "sg-09d19538187df66cf",
+                  "name": "asia"
+                },
+                {
+                  "id": "sg-0ae266faddaef17e9",
+                  "name": "packer-ssh-access"
+                }
+              ]
+            }
+          }
+        }
+      ]
+    }
+  ],
   "tenant_name": "AWS-TESTING",
   "id": "123456789012",
-  "cloud": "aws",
+  "cloud": "AWS",
   "activated_regions": [
     "eu-central-1",
     "eu-north-1",
     "eu-west-1",
     "eu-west-3"
   ],
-  "from": "2024-11-03T00:00:00Z",
-  "to": "2024-11-10T00:00:00Z",
-  "last_scan_date": "2024-11-04T00:00:00Z",
-  "outdated_tenants": [],
-  "data": [],
-  "externalData": false,
-  "report_type": "FINOPS"
+  "last_scan_date": "2024-12-12T00:38:12.463910Z"
 }