From 380c67bee4437eb7a132870665e18de534251217 Mon Sep 17 00:00:00 2001 From: AWS Date: Wed, 7 Aug 2024 19:53:06 +0000 Subject: [PATCH] Release: 1.13.1 --- VERSION | 2 +- .../buildspecs/aft-lambda-layer.yml | 1 + .../account_provisioning_framework.py | 5 ++-- .../aft_common/account_request_framework.py | 6 +++-- .../aft-lambda-layer/aft_common/aft_utils.py | 27 ++++++++++++++----- .../aft_common/codepipeline.py | 3 ++- sources/aft-lambda-layer/aft_common/ddb.py | 7 +++-- .../aft_common/notifications.py | 4 ++- sources/aft-lambda-layer/aft_common/sqs.py | 4 +-- sources/aft-lambda-layer/pyproject.toml | 2 +- .../aft_builder/codebuild_trigger.py | 20 +++++++++----- .../aft_customizations_execute_pipeline.py | 4 ++- .../aft_customizations_identify_targets.py | 12 ++++++--- 13 files changed, 67 insertions(+), 30 deletions(-) diff --git a/VERSION b/VERSION index feaae22b..b50dd27d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.0 +1.13.1 diff --git a/modules/aft-lambda-layer/buildspecs/aft-lambda-layer.yml b/modules/aft-lambda-layer/buildspecs/aft-lambda-layer.yml index d5338c2a..118d53d8 100644 --- a/modules/aft-lambda-layer/buildspecs/aft-lambda-layer.yml +++ b/modules/aft-lambda-layer/buildspecs/aft-lambda-layer.yml @@ -34,6 +34,7 @@ phases: - python3 -m pip install virtualenv - python3 -m venv .venv - . .venv/bin/activate + - python3 -m pip install --upgrade 'setuptools>=70.0.0' - python3 -m pip install ./aws-aft-core-framework/sources/aft-lambda-layer build: commands: diff --git a/sources/aft-lambda-layer/aft_common/account_provisioning_framework.py b/sources/aft-lambda-layer/aft_common/account_provisioning_framework.py index 6642f7a9..267ca00e 100644 --- a/sources/aft-lambda-layer/aft_common/account_provisioning_framework.py +++ b/sources/aft-lambda-layer/aft_common/account_provisioning_framework.py @@ -11,6 +11,7 @@ import aft_common.constants import aft_common.ssm from aft_common import ddb +from aft_common.aft_utils import sanitize_input_for_logging from aft_common.auth import AuthClient from aft_common.organizations import OrganizationsAgent from boto3.session import Session @@ -259,8 +260,8 @@ def persist_metadata( logger.info(item) response = ddb.put_ddb_item(session, metadata_table_name, item) - - logger.info(response) + sanitized_response = sanitize_input_for_logging(response) + logger.info(sanitized_response) return response diff --git a/sources/aft-lambda-layer/aft_common/account_request_framework.py b/sources/aft-lambda-layer/aft_common/account_request_framework.py index c9218985..9c58967f 100644 --- a/sources/aft-lambda-layer/aft_common/account_request_framework.py +++ b/sources/aft-lambda-layer/aft_common/account_request_framework.py @@ -112,7 +112,8 @@ def put_audit_record( item["ddb_event_name"] = {"S": event_name} logger.info("Inserting item into " + table + " table: " + str(item)) response = dynamodb.put_item(TableName=table, Item=item) - logger.info(response) + sanitized_response = utils.sanitize_input_for_logging(response) + logger.info(sanitized_response) return response @@ -210,7 +211,8 @@ def create_new_account( ), ProvisionToken=str(uuid.uuid1()), ) - logger.info(response) + sanitized_response = utils.sanitize_input_for_logging(response) + logger.info(sanitized_response) return response diff --git a/sources/aft-lambda-layer/aft_common/aft_utils.py b/sources/aft-lambda-layer/aft_common/aft_utils.py index 3022329f..b5060c89 100644 --- a/sources/aft-lambda-layer/aft_common/aft_utils.py +++ b/sources/aft-lambda-layer/aft_common/aft_utils.py @@ -3,6 +3,7 @@ # import logging import random +import re import time from functools import wraps from typing import ( @@ -78,14 +79,17 @@ def wrapper(*args: Optional[Tuple[Any]], **kwargs: Optional[Dict[str, Any]]) -> return func(*args, **kwargs) except ClientError as e: if e.response["Error"]["Code"] in BOTO3_CLIENT_ERROR_THROTTLING_CODES: + sanitized_max_requests = sanitize_input_for_logging(max_requests) + sanitized_retry_sleep_sec = sanitize_input_for_logging( + retry_sleep_sec + ) if requests >= max_requests: logger.info( - f"Exceeded max fresh-request retry attempts ({max_requests})" + f"Exceeded max fresh-request retry attempts ({sanitized_max_requests})" ) raise e - logger.info( - f"Exceeded max boto3 retries on previous request. Retrying with fresh request in {retry_sleep_sec} seconds." + f"Exceeded max boto3 retries on previous request. Retrying with fresh request in {sanitized_retry_sleep_sec} seconds." ) requests += 1 time.sleep(retry_sleep_sec) @@ -124,14 +128,16 @@ def invoke_lambda( payload: Union[bytes, IO[bytes], StreamingBody], ) -> InvocationResponseTypeDef: client: LambdaClient = session.client("lambda") - logger.info(f"Invoking Lambda: {function_name}") + sanitized_function_name = sanitize_input_for_logging(function_name) + logger.info(f"Invoking Lambda: {sanitized_function_name}") response = client.invoke( FunctionName=function_name, InvocationType="Event", LogType="Tail", Payload=payload, ) - logger.info(response) + sanitized_response = sanitize_input_for_logging(response) + logger.info(sanitized_response) return response @@ -153,7 +159,8 @@ def invoke_step_function( ) -> StartExecutionOutputTypeDef: client: SFNClient = session.client("stepfunctions") sfn_arn = build_sfn_arn(session, sfn_name) - logger.info("Starting SFN execution of " + sfn_arn) + sanitized_sfn_arn = sanitize_input_for_logging(sfn_arn) + logger.info("Starting SFN execution of " + sanitized_sfn_arn) response = client.start_execution(stateMachineArn=sfn_arn, input=input) logger.debug(response) return response @@ -196,3 +203,11 @@ def yield_batches_from_list( while idx < len(input): yield input[idx : idx + batch_size] idx += batch_size + + +def sanitize_input_for_logging(input: Any) -> str: + """ + Sanitize the input string by replacing newline characters, tabs with their literal string representations. + """ + input_str = str(input) + return input_str.encode("unicode_escape").decode() diff --git a/sources/aft-lambda-layer/aft_common/codepipeline.py b/sources/aft-lambda-layer/aft_common/codepipeline.py index c665ebfe..57e7c9dc 100644 --- a/sources/aft-lambda-layer/aft_common/codepipeline.py +++ b/sources/aft-lambda-layer/aft_common/codepipeline.py @@ -76,7 +76,8 @@ def execute_pipeline(session: Session, account_id: str) -> None: if not pipeline_is_running(session, name): logger.info("Executing pipeline - " + name) response = client.start_pipeline_execution(name=name) - logger.info(response) + sanitized_response = utils.sanitize_input_for_logging(response) + logger.info(sanitized_response) else: logger.info("Pipeline is currently running") diff --git a/sources/aft-lambda-layer/aft_common/ddb.py b/sources/aft-lambda-layer/aft_common/ddb.py index 7ed64f90..5abc139a 100644 --- a/sources/aft-lambda-layer/aft_common/ddb.py +++ b/sources/aft-lambda-layer/aft_common/ddb.py @@ -4,6 +4,7 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Optional +from aft_common.aft_utils import sanitize_input_for_logging from boto3.dynamodb.types import TypeDeserializer from boto3.session import Session @@ -42,7 +43,8 @@ def put_ddb_item( logger.info(f"Inserting item into {table_name} table: {str(item)}") response = table.put_item(Item=item) - logger.info(response) + sanitized_response = sanitize_input_for_logging(response) + logger.info(sanitized_response) return response @@ -54,7 +56,8 @@ def delete_ddb_item( logger.info(f"Deleting item with key: {primary_key} from: {table_name} table") response = table.delete_item(Key=primary_key) - logger.info(response) + sanitized_response = sanitize_input_for_logging(response) + logger.info(sanitized_response) return response diff --git a/sources/aft-lambda-layer/aft_common/notifications.py b/sources/aft-lambda-layer/aft_common/notifications.py index 8a23b62f..dbfc1e35 100644 --- a/sources/aft-lambda-layer/aft_common/notifications.py +++ b/sources/aft-lambda-layer/aft_common/notifications.py @@ -4,6 +4,7 @@ import logging from typing import TYPE_CHECKING +from aft_common.aft_utils import sanitize_input_for_logging from aft_common.constants import SSM_PARAM_SNS_FAILURE_TOPIC_ARN from aft_common.ssm import get_ssm_parameter_value from boto3.session import Session @@ -27,7 +28,8 @@ def send_sns_message( logger.info("Sending SNS Message") client: SNSClient = session.client("sns") response = client.publish(TopicArn=topic, Message=sns_message, Subject=subject) - logger.info(response) + sanitized_response = sanitize_input_for_logging(response) + logger.info(sanitized_response) return response diff --git a/sources/aft-lambda-layer/aft_common/sqs.py b/sources/aft-lambda-layer/aft_common/sqs.py index 803208e2..d3445f35 100644 --- a/sources/aft-lambda-layer/aft_common/sqs.py +++ b/sources/aft-lambda-layer/aft_common/sqs.py @@ -75,7 +75,7 @@ def send_sqs_message( MessageDeduplicationId=unique_id, MessageGroupId=unique_id, ) - - logger.info(response) + sanitized_response = utils.sanitize_input_for_logging(response) + logger.info(sanitized_response) return response diff --git a/sources/aft-lambda-layer/pyproject.toml b/sources/aft-lambda-layer/pyproject.toml index 29f67a2e..414292d3 100644 --- a/sources/aft-lambda-layer/pyproject.toml +++ b/sources/aft-lambda-layer/pyproject.toml @@ -26,7 +26,7 @@ classifiers=[ dependencies = [ "boto3 == 1.28.17", "botocore == 1.31.17", - "requests == 2.31.0", + "requests == 2.32.2", "jsonschema == 4.3.2", ] diff --git a/src/aft_lambda/aft_builder/codebuild_trigger.py b/src/aft_lambda/aft_builder/codebuild_trigger.py index f02f3283..8ecf00ea 100644 --- a/src/aft_lambda/aft_builder/codebuild_trigger.py +++ b/src/aft_lambda/aft_builder/codebuild_trigger.py @@ -4,6 +4,7 @@ import datetime import inspect import logging +import re import time from typing import Any, Dict, TypedDict @@ -26,9 +27,13 @@ def lambda_handler(event: Dict[str, Any], context: Dict[str, Any]) -> LayerBuild codebuild_project_name = event["codebuild_project_name"] job_id = client.start_build(projectName=codebuild_project_name)["build"]["id"] - - logger.info(f"Started build project {codebuild_project_name} job {job_id}") - + sanitized_codebuild_project_name = re.sub( + r"[^a-zA-Z0-9-_]", "", codebuild_project_name + ) + sanitized_job_id = re.sub(r"[^a-zA-Z0-9-_]", "", job_id) + logger.info( + f"Started build project {sanitized_codebuild_project_name} job {sanitized_job_id}" + ) # Wait at least 30 seconds for the build to initialize time.sleep(30) @@ -43,13 +48,14 @@ def lambda_handler(event: Dict[str, Any], context: Dict[str, Any]) -> LayerBuild time.sleep(10) continue elif job_status == "SUCCEEDED": - logger.info(f"Build job {job_id} completed successfully") + logger.info(f"Build job {sanitized_job_id} completed successfully") return {"Status": 200} else: - logger.info(f"Build {job_id} failed - non-success terminal status") + logger.info( + f"Build {sanitized_job_id} failed - non-success terminal status" + ) raise Exception(f"Build {job_id} failed - non-success terminal status") - - logger.info(f"Build {job_id} failed - time out") + logger.info(f"Build {sanitized_job_id} failed - time out") raise Exception(f"Build {job_id} failed - time out") except Exception as error: diff --git a/src/aft_lambda/aft_customizations/aft_customizations_execute_pipeline.py b/src/aft_lambda/aft_customizations/aft_customizations_execute_pipeline.py index 9563c4ff..2d1c388e 100644 --- a/src/aft_lambda/aft_customizations/aft_customizations_execute_pipeline.py +++ b/src/aft_lambda/aft_customizations/aft_customizations_execute_pipeline.py @@ -8,6 +8,7 @@ import aft_common.ssm from aft_common import constants as utils from aft_common import notifications +from aft_common.aft_utils import sanitize_input_for_logging from aft_common.codepipeline import execute_pipeline from aft_common.logger import configure_aft_logger, customization_request_logger from boto3.session import Session @@ -38,7 +39,8 @@ def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, A execute_pipeline(session, str(account_id)) accounts.remove(account_id) logger.info("Accounts remaining to be executed - ") - logger.info(accounts) + sanitized_accounts = sanitize_input_for_logging(accounts) + logger.info(sanitized_accounts) return {"number_pending_accounts": len(accounts), "pending_accounts": accounts} except Exception as error: diff --git a/src/aft_lambda/aft_customizations/aft_customizations_identify_targets.py b/src/aft_lambda/aft_customizations/aft_customizations_identify_targets.py index 6bfa94b5..afcd6162 100644 --- a/src/aft_lambda/aft_customizations/aft_customizations_identify_targets.py +++ b/src/aft_lambda/aft_customizations/aft_customizations_identify_targets.py @@ -11,6 +11,7 @@ build_account_customization_payload, get_account_request_record, ) +from aft_common.aft_utils import sanitize_input_for_logging from aft_common.auth import AuthClient from aft_common.customizations import ( get_excluded_accounts, @@ -63,14 +64,16 @@ def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, A target_account_info = [] for account_id in target_accounts: - logger.info(f"Building customization payload for {account_id}") - + sanitized_account_id = sanitize_input_for_logging(account_id) + logger.info( + f"Building customization payload for {sanitized_account_id}" + ) try: account_email = orgs_agent.get_account_email_from_id(account_id) except ClientError as error: if error.response["Error"]["Code"] == "AccountNotFoundException": logger.info( - f"Account with ID {account_id} does not exist or is suspended - ignoring" + f"Account with ID {sanitized_account_id} does not exist or is suspended - ignoring" ) target_accounts.remove(account_id) continue @@ -87,7 +90,8 @@ def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, A account_request=account_request, control_tower_event={}, ) - logger.info(f"Successfully generated payload: {account_payload}") + sanitized_payload = sanitize_input_for_logging(account_payload) + logger.info(f"Successfully generated payload: {sanitized_payload}") target_account_info.append(account_payload) return {