From f515b2b53b684ea1d0a8ca7e0c81efe8a413cf78 Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Fri, 28 Jun 2024 09:27:47 -0400 Subject: [PATCH] fix(aws): parallelize functions per resource (#4323) --- .../services/cloudwatch/cloudwatch_service.py | 28 ++++--- .../services/codebuild/codebuild_service.py | 75 +++++++------------ 2 files changed, 41 insertions(+), 62 deletions(-) diff --git a/prowler/providers/aws/services/cloudwatch/cloudwatch_service.py b/prowler/providers/aws/services/cloudwatch/cloudwatch_service.py index 2166779e2b..e2ba648843 100644 --- a/prowler/providers/aws/services/cloudwatch/cloudwatch_service.py +++ b/prowler/providers/aws/services/cloudwatch/cloudwatch_service.py @@ -95,7 +95,7 @@ def __init__(self, provider): 1000 # The threshold for number of events to return per log group. ) self.__threading_call__(self.__get_log_events__) - self.__list_tags_for_resource__() + self.__threading_call__(self.__list_tags_for_resource__, self.log_groups) def __describe_metric_filters__(self, regional_client): logger.info("CloudWatch Logs - Describing metric filters...") @@ -214,21 +214,19 @@ def __get_log_events__(self, regional_client): f"CloudWatch Logs - Finished retrieving log events in {regional_client.region}..." ) - def __list_tags_for_resource__(self): - logger.info("CloudWatch Logs - List Tags...") + def __list_tags_for_resource__(self, log_group): + logger.info(f"CloudWatch Logs - List Tags for Log Group {log_group.name}...") try: - for log_group in self.log_groups: - try: - regional_client = self.regional_clients[log_group.region] - response = regional_client.list_tags_log_group( - logGroupName=log_group.name - )["tags"] - log_group.tags = [response] - except ClientError as error: - if error.response["Error"]["Code"] == "ResourceNotFoundException": - log_group.tags = [] - - continue + regional_client = self.regional_clients[log_group.region] + response = regional_client.list_tags_for_resource( + resourceArn=log_group.arn + )["tags"] + log_group.tags = [response] + except ClientError as error: + if error.response["Error"]["Code"] == "ResourceNotFoundException": + logger.warning( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) except Exception as error: logger.error( f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/providers/aws/services/codebuild/codebuild_service.py b/prowler/providers/aws/services/codebuild/codebuild_service.py index ef889fe679..db0f080d66 100644 --- a/prowler/providers/aws/services/codebuild/codebuild_service.py +++ b/prowler/providers/aws/services/codebuild/codebuild_service.py @@ -15,9 +15,11 @@ def __init__(self, provider): super().__init__(__class__.__name__, provider) self.projects = {} self.__threading_call__(self.__list_projects__) - self.__threading_call__(self.__list_builds_for_project__) - self.__threading_call__(self.__batch_get_builds__) - self.__threading_call__(self.__batch_get_projects__) + self.__threading_call__( + self.__list_builds_for_project__, self.projects.values() + ) + self.__threading_call__(self.__batch_get_builds__, self.projects.values()) + self.__threading_call__(self.__batch_get_projects__, self.projects.values()) def __list_projects__(self, regional_client): logger.info("Codebuild - Listing projects...") @@ -40,64 +42,43 @@ def __list_projects__(self, regional_client): f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __list_builds_for_project__(self, regional_client): + def __list_builds_for_project__(self, project): logger.info("Codebuild - Listing builds...") try: - for project in self.projects.values(): - if project.region == regional_client.region: - try: - build_ids = regional_client.list_builds_for_project( - projectName=project.name - ).get("ids", []) - if len(build_ids) > 0: - project.last_build = Build(id=build_ids[0]) - except Exception as error: - logger.error( - f"{regional_client.region}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + regional_client = self.regional_clients[project.region] + build_ids = regional_client.list_builds_for_project( + projectName=project.name + ).get("ids", []) + if len(build_ids) > 0: + project.last_build = Build(id=build_ids[0]) except Exception as error: logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + f"{project.region}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __batch_get_builds__(self, regional_client): + def __batch_get_builds__(self, project): logger.info("Codebuild - Getting builds...") try: - for project in self.projects.values(): - if ( - project.region == regional_client.region - and project.last_build - and project.last_build.id - ): - try: - builds_by_id = regional_client.batch_get_builds( - ids=[project.last_build.id] - ).get("builds", []) - if len(builds_by_id) > 0: - project.last_invoked_time = builds_by_id[0].get("endTime") - except Exception as error: - logger.error( - f"{regional_client.region}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + if project.last_build and project.last_build.id: + regional_client = self.regional_clients[project.region] + builds_by_id = regional_client.batch_get_builds( + ids=[project.last_build.id] + ).get("builds", []) + if len(builds_by_id) > 0: + project.last_invoked_time = builds_by_id[0].get("endTime") except Exception as error: logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + f"{regional_client.region}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __batch_get_projects__(self, regional_client): + def __batch_get_projects__(self, project): logger.info("Codebuild - Getting projects...") try: - for project in self.projects.values(): - if project.region == regional_client.region: - try: - project_source = regional_client.batch_get_projects( - names=[project.name] - )["projects"][0]["source"] - project.buildspec = project_source.get("buildspec", "") - except Exception as error: - logger.error( - f"{regional_client.region}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + regional_client = self.regional_clients[project.region] + project_source = regional_client.batch_get_projects(names=[project.name])[ + "projects" + ][0]["source"] + project.buildspec = project_source.get("buildspec", "") except Exception as error: logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"