From 6616aa37940e65777d914817a247356be27068be Mon Sep 17 00:00:00 2001 From: Tarek Abdunabi Date: Wed, 30 Nov 2022 19:30:59 -0800 Subject: [PATCH] Update to version v2.1.0 --- CHANGELOG.md | 10 + deployment/build-s3-dist.sh | 8 +- .../lambdas/pipeline_orchestration/index.py | 60 +- .../pipeline_orchestration/shared/helper.py | 10 + .../solution_model_card.py | 666 ++++++++++++++++++ .../tests/test_pipeline_orchestration.py | 60 ++ .../tests/test_solution_model_card.py | 365 ++++++++++ .../byom/autopilot_training_pipeline.py | 5 - .../byom/lambdas/batch_transform/main.py | 2 +- .../create_baseline_job/baselines_helper.py | 28 + .../tests/test_create_data_baseline.py | 66 +- .../lambdas/create_update_cf_stackset/main.py | 2 +- .../lambdas/sagemaker_layer/requirements.txt | 5 +- .../byom/model_training_pipeline.py | 1 - .../byom/multi_account_codepipeline.py | 3 - .../pipeline_definitions/deploy_actions.py | 15 +- .../byom/pipeline_definitions/helpers.py | 2 +- .../byom/pipeline_definitions/iam_policies.py | 48 +- .../sagemaker_model_monitor_construct.py | 5 +- .../byom/single_account_codepipeline.py | 1 - source/lib/mlops_orchestrator_stack.py | 8 +- source/requirements-test.txt | 11 +- 22 files changed, 1327 insertions(+), 54 deletions(-) create mode 100644 source/lambdas/pipeline_orchestration/solution_model_card.py create mode 100644 source/lambdas/pipeline_orchestration/tests/test_solution_model_card.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ca0fa1..c221015 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.0] - 2022-11-30 + +### Added + +- Integration with Amazon SageMaker Model Card and Model Dashboard features to allow customers to perform model card operations. All Amazon SageMaker resources (models, endpoints, training jobs, and model monitors) created by the solution will show up on the SageMaker Model Dashboard. + +### Fixed + +- Missing AWS IAM Role permissions used by the Amazon SageMaker Clarify Model Bias Monitor and Amazon SageMaker Clarify Model Explainability Monitor scheduling jobs. + ## [2.0.1] - 2022-08-12 ### Updated diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh index dd5c0e4..6e24e3e 100755 --- a/deployment/build-s3-dist.sh +++ b/deployment/build-s3-dist.sh @@ -69,8 +69,12 @@ echo "cd $source_dir" cd $source_dir # setup lambda layers (building sagemaker layer using lambda build environment for python 3.8) -echo 'docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install --upgrade -r requirements.txt -t ./python; exit"' -docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install --upgrade -r requirements.txt -t ./python; exit" +echo 'docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit"' +docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit" + +# Remove tests and cache stuff (to reduce size) +find "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer/python -type d -name "tests" -exec rm -rfv {} + +find "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer/python -type d -name "__pycache__" -exec rm -rfv {} + echo "python3 -m venv .venv-prod" python3 -m venv .venv-prod diff --git a/source/lambdas/pipeline_orchestration/index.py b/source/lambdas/pipeline_orchestration/index.py index 45765c2..cb03c1e 100644 --- a/source/lambdas/pipeline_orchestration/index.py +++ b/source/lambdas/pipeline_orchestration/index.py @@ -11,14 +11,13 @@ # and limitations under the License. # # ##################################################################################################################### import json -from json import JSONEncoder import os -import datetime from botocore.client import BaseClient +from sagemaker.session import Session from typing import Dict, Any, List, Union from shared.wrappers import BadRequest, api_exception_handler from shared.logger import get_logger -from shared.helper import get_client +from shared.helper import get_client, DateTimeEncoder from lambda_helpers import ( validate, template_url, @@ -28,31 +27,40 @@ format_template_parameters, create_template_zip_file, ) +from solution_model_card import SolutionModelCardAPIs cloudformation_client = get_client("cloudformation") codepipeline_client = get_client("codepipeline") s3_client = get_client("s3") +sm_client = get_client("sagemaker") +sagemaker_session = Session(sagemaker_client=sm_client) + logger = get_logger(__name__) content_type = "plain/text" - -# subclass JSONEncoder to be able to convert pipeline status to json -class DateTimeEncoder(JSONEncoder): - # Override the default method - def default(self, obj): - if isinstance(obj, (datetime.date, datetime.datetime)): - return obj.isoformat() +model_card_operations = [ + "create_model_card", + "update_model_card", + "describe_model_card", + "delete_model_card", + "export_model_card", + "list_model_cards", +] @api_exception_handler def handler(event: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: + if "httpMethod" in event and event["httpMethod"] == "POST": # Lambda is being invoked from API Gateway - if event["path"] == "/provisionpipeline": - return provision_pipeline(json.loads(event["body"])) + event_body = json.loads(event["body"]) + if event["path"] == "/provisionpipeline" and event_body.get("pipeline_type") in model_card_operations: + return provision_model_card(event_body, sagemaker_session) + elif event["path"] == "/provisionpipeline": + return provision_pipeline(event_body) elif event["path"] == "/pipelinestatus": - return pipeline_status(json.loads(event["body"])) + return pipeline_status(event_body) else: raise BadRequest("Unacceptable event path. Path must be /provisionpipeline or /pipelinestatus") elif "pipeline_type" in event: # Lambda is being invoked from codepipeline/codebuild @@ -64,6 +72,28 @@ def handler(event: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: ) +def provision_model_card(event: Dict[str, Any], sagemaker_session: Session) -> Dict[str, Any]: + pipeline_type = event.get("pipeline_type") + if pipeline_type == "create_model_card": + return SolutionModelCardAPIs(event, sagemaker_session).create() + elif pipeline_type == "update_model_card": + return SolutionModelCardAPIs(event, sagemaker_session).update() + elif pipeline_type == "describe_model_card": + return SolutionModelCardAPIs(event, sagemaker_session).describe() + elif pipeline_type == "delete_model_card": + return SolutionModelCardAPIs(event, sagemaker_session).delete() + elif pipeline_type == "export_model_card": + return SolutionModelCardAPIs(event, sagemaker_session).export_to_pdf( + s3_output_path=f"s3://{os.environ['ASSETS_BUCKET']}/model_card_exports" + ) + elif pipeline_type == "list_model_cards": + return SolutionModelCardAPIs(event, sagemaker_session).list_model_cards() + else: + raise BadRequest( + "pipeline_type must be on of create_model_card|update_model_card|describe_model_card|delete_model_card|export_model_card" + ) + + def provision_pipeline( event: Dict[str, Any], client: BaseClient = cloudformation_client, @@ -72,7 +102,7 @@ def provision_pipeline( """ provision_pipeline takes the lambda event object and creates a cloudformation stack - :event: event object from lambda function. It must containe: pipeline_type, custom_model_container, + :event: event object from lambda function. It must contain: pipeline_type, custom_model_container, model_framework, model_framework_version, model_name, model_artifact_location, training_data, inference_instance, inference_type, batch_inference_data :client: boto3 cloudformation client. Not needed, it is only added for unit testing purpose @@ -135,7 +165,7 @@ def provision_pipeline( s3_client, ) - logger.info("New pipelin stack created") + logger.info("New pipeline stack created") logger.info(stack_response) response = { "statusCode": 200, diff --git a/source/lambdas/pipeline_orchestration/shared/helper.py b/source/lambdas/pipeline_orchestration/shared/helper.py index 4de616e..4f2bb42 100644 --- a/source/lambdas/pipeline_orchestration/shared/helper.py +++ b/source/lambdas/pipeline_orchestration/shared/helper.py @@ -13,6 +13,8 @@ import boto3 import json import os +import datetime +from json import JSONEncoder from botocore.config import Config from shared.logger import get_logger @@ -38,3 +40,11 @@ def get_client(service_name, config=CLIENT_CONFIG): def reset_client(): global _helpers_service_clients _helpers_service_clients = dict() + + +# subclass JSONEncoder to be able to convert pipeline status to json +class DateTimeEncoder(JSONEncoder): + # Override the default method + def default(self, obj): + if isinstance(obj, (datetime.date, datetime.datetime)): + return obj.isoformat() \ No newline at end of file diff --git a/source/lambdas/pipeline_orchestration/solution_model_card.py b/source/lambdas/pipeline_orchestration/solution_model_card.py new file mode 100644 index 0000000..c72a94e --- /dev/null +++ b/source/lambdas/pipeline_orchestration/solution_model_card.py @@ -0,0 +1,666 @@ +# ##################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. A copy of the License is located at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # +# and limitations under the License. # +# ##################################################################################################################### +from typing import Optional, Union, List, Dict, Any +import json +import boto3 +from sagemaker.session import Session +from sagemaker.model_card import ( + Environment, + ModelOverview, + IntendedUses, + ObjectiveFunction, + Metric, + TrainingDetails, + MetricGroup, + EvaluationJob, + AdditionalInformation, + ModelCard, + Function, + TrainingJobDetails, + RiskRatingEnum, + ObjectiveFunctionEnum, + FacetEnum, + MetricTypeEnum, + ModelCardStatusEnum, + EvaluationMetricTypeEnum, +) + +from sagemaker.model_card.model_card import TrainingMetric +from shared.helper import DateTimeEncoder +from shared.logger import get_logger +from shared.wrappers import exception_handler + +logger = get_logger(__name__) + + +class SolutionModelCardHelpers: + @classmethod + @exception_handler + def get_environment(cls, container_image: List[str]) -> Union[Environment, None]: + """Initialize an Environment object. + + Args: + container_image (list[str]): A list of SageMaker training/inference image URIs. The maximum list length is 15. + """ + return Environment(container_image=container_image) if container_image else None + + @classmethod + @exception_handler + def list_model_cards(cls, sagemaker_session: Session) -> Dict[str, Any]: + """List all model cards in the SageMaker service. + + Args: + sagemaker_session (sagemaker.session.Session): SageMaker session. + Returns: + a Python dictionary with all model cards + """ + return sagemaker_session.sagemaker_client.list_model_cards() + + @classmethod + @exception_handler + def get_model_overview( + cls, model_name: Optional[str] = None, sagemaker_session: Session = None, **kwargs + ) -> ModelOverview: + """Initialize a Model Overview object. + + Args: + model_name (str, optional): A unique name for the model (default: None). + sagemaker_session (sagemaker.session.Session): SageMaker session (default: None). + model_id (str, optional): A SageMaker Model ARN or non-SageMaker Model ID (default: None). + kwargs: possible values: + model_description (str, optional): A description of the model (default: None). + model_version (int or float, optional): The version of the model (default: None). + problem_type (str, optional): The type of problem that the model solves. For example, "Binary Classification", "Multiclass Classification", "Linear Regression", "Computer Vision", or "Natural Language Processing" (default: None). + algorithm_type (str, optional): The algorithm used to solve the problem type (default: None). + model_creator (str, optional): The organization, research group, or authors that created the model (default: None). + model_owner (str, optional): The individual or group that maintains the model in your organization (default: None). + model_artifact (List[str], optional): A list of model artifact location URIs. The maximum list size is 15. (default: None). + inference_environment (Environment, optional): An overview of the model's inference environment (default: None). + Returns: + sagemaker.model_card.ModelOverview + """ + # automatically get ModelOverview from model name is provided + if model_name: + return ModelOverview.from_model_name(model_name=model_name, sagemaker_session=sagemaker_session, **kwargs) + else: + # update inference_environment if provided + inference_environment = kwargs.get("inference_environment") + kwargs.update( + {"inference_environment": cls.get_environment(inference_environment) if inference_environment else None} + ) + return ModelOverview(**kwargs) + + @classmethod + @exception_handler + def get_intended_uses( + cls, + purpose_of_model: Optional[str] = None, + intended_uses: Optional[str] = None, + factors_affecting_model_efficiency: Optional[str] = None, + risk_rating: Optional[Union[RiskRatingEnum, str]] = RiskRatingEnum.UNKNOWN, + explanations_for_risk_rating: Optional[str] = None, + ) -> IntendedUses: + """Initialize an Intended Uses object. + + Args: + purpose_of_model (str, optional): The general purpose of this model (default: None). + intended_uses (str, optional): The intended use cases for this model (default: None). + factors_affecting_model_efficiency (str, optional): Factors affecting model efficacy (default: None). + risk_rating (RiskRatingEnum or str, optional): Your organization's risk rating for this model. It is highly recommended to use sagemaker.model_card.RiskRatingEnum. Possible values include: ``RiskRatingEnum.HIGH`` ("High"), ``RiskRatingEnum.LOW`` ("Low"), ``RiskRatingEnum.MEDIUM`` ("Medium"), or ``RiskRatingEnum.UNKNOWN`` ("Unknown"). Defaults to ``RiskRatingEnum.UNKNOWN``. + explanations_for_risk_rating (str, optional): An explanation of why your organization categorizes this model with this risk rating (default: None). + Returns: + sagemaker.model_card.IntendedUses + """ + return IntendedUses( + purpose_of_model=purpose_of_model, + intended_uses=intended_uses, + factors_affecting_model_efficiency=factors_affecting_model_efficiency, + risk_rating=risk_rating, + explanations_for_risk_rating=explanations_for_risk_rating, + ) + + @classmethod + @exception_handler + def get_objective_function( + cls, + function: Optional[Union[ObjectiveFunctionEnum, str]] = None, + facet: Optional[Union[FacetEnum, str]] = None, + condition: Optional[str] = None, + notes: Optional[str] = None, + ) -> ObjectiveFunction: + """Initialize an Objective Function object. + + Args: + function (ObjectiveFunctionEnum or str, optional): The optimization direction of the model's objective function. It is highly recommended to use sagemaker.model_card.ObjectiveFunctionEnum. Possible values include: ``ObjectiveFunctionEnum.MAXIMIZE`` ("Maximize") or ``ObjectiveFunctionEnum.MINIMIZE`` ("Minimize") (default: None). + facet (FacetEnum or str, optional): The metric of the model's objective function. For example, `loss` or `rmse`. It is highly recommended to use sagemaker.model_card.FacetEnum. Possible values include:, ``FacetEnum.ACCURACY`` ("Accuracy"), ``FacetEnum.AUC`` ("AUC"), ``FacetEnum.LOSS`` ("Loss"), ``FacetEnum.MAE`` ("MAE"), or ``FacetEnum.RMSE`` ("RMSE") (default: None). + condition (str, optional): An optional description of any conditions of your objective function metric (default: None). + notes (str, optional): Notes about the objective function, including other considerations for possible objective functions (default: None). + Returns: + sagemaker.model_card.ObjectiveFunction + """ + return ObjectiveFunction(function=Function(function=function, facet=facet, condition=condition), notes=notes) + + @classmethod + @exception_handler + def get_metric( + cls, + name: str, + type: Union[MetricTypeEnum, str], + value: Union[int, float, str, bool, List], + notes: Optional[str] = None, + x_axis_name: Optional[Union[str, list]] = None, + y_axis_name: Optional[Union[str, list]] = None, + ) -> Metric: + """Initialize a Metric object. + + Args: + name (str): The name of the metric. + type (str or MetricTypeEnum): It is highly recommended to use sagemaker.model_card.MetricTypeEnum. Possible values include: + ``MetricTypeEnum.BAR_CHART`` ("bar_char"), ``MetricTypeEnum.BOOLEAN`` ("boolean"), + ``MetricTypeEnum.LINEAR_GRAPH`` ("linear_graph"), ``MetricTypeEnum.MATRIX`` ("matrix"), ``MetricTypeEnum.NUMBER`` + ("number"), or ``MetricTypeEnum.STRING`` ("string"). + value (int or float or str or bool or List): The datatype of the metric. The metric's `value` must be compatible with the metric's `type`. + notes (str, optional): Any notes to add to the metric (default: None). + x_axis_name (str, optional): The name of the x axis (default: None). + y_axis_name (str, optional): The name of the y axis (default: None). + Returns: + sagemaker.model_card.Metric + """ + return Metric(name=name, type=type, value=value, notes=notes, x_axis_name=x_axis_name, y_axis_name=y_axis_name) + + @classmethod + @exception_handler + def get_metric_group(cls, name: str, metric_data: Optional[List[Dict[str, Any]]] = None) -> MetricGroup: + """Initialize a Metric Group object. + + Args: + name (str): The metric group name. + metric_data (List[Dict[str, Any]]): A list of dictionaries representing metrics. + Returns: + sagemaker.model_card.MetricGroup + """ + return MetricGroup( + name=name, metric_data=[cls.get_metric(**metric) for metric in metric_data] if metric_data else None + ) + + @classmethod + @exception_handler + def get_training_metric( + cls, + name: str, + value: Union[int, float], + notes: Optional[str] = None, + ) -> TrainingMetric: + """Initialize a TrainingMetric object. + + Args: + name (str): The metric name. + value (int or float): The metric value. + notes (str, optional): Notes on the metric (default: None). + Returns: + sagemaker.model_card.TrainingMetric + """ + return TrainingMetric(name=name, value=value, notes=notes) + + @classmethod + @exception_handler + def get_training_job_details( + cls, + training_arn: Optional[str] = None, + training_datasets: Optional[List[str]] = None, + training_environment: Optional[List[str]] = None, + training_metrics: Optional[List[Dict[str, Any]]] = None, + user_provided_training_metrics: Optional[List[Dict[str, Any]]] = None, + ) -> TrainingJobDetails: + """Initialize a Training Job Details object. + + Args: + training_arn (str, optional): The SageMaker training job Amazon Resource Name (ARN) (default: None). + training_datasets (List[str], optional): The location of the datasets used to train the model. The maximum list size is 15. (default: None). + training_environment (Environment, optional): The SageMaker training image URI. (default: None). + training_metrics (List[Dict[str, Any]], optional): SageMaker training job results. The maximum `training_metrics` list length is 50 (default: None). + user_provided_training_metrics (List[Dict[str, Any]], optional): Custom training job results. The maximum `user_provided_training_metrics` list length is 50 (default: None). + Returns: + sagemaker.model_card.TrainingJobDetails + """ + return TrainingJobDetails( + training_arn=training_arn, + training_datasets=training_datasets, + training_environment=cls.get_environment(training_environment), + training_metrics=[cls.get_training_metric(**metric) for metric in training_metrics] + if training_metrics + else None, + user_provided_training_metrics=[ + cls.get_training_metric(**metric) for metric in user_provided_training_metrics + ] + if user_provided_training_metrics + else None, + ) + + @classmethod + @exception_handler + def get_training_details( + cls, + model_name: Optional[str] = None, + training_job_name: Optional[str] = None, + objective_function: Optional[Dict[str, Any]] = None, + training_observations: Optional[str] = None, + training_job_details: Optional[Dict[str, Any]] = None, + sagemaker_session: Session = None, + ) -> TrainingDetails: + """Initialize a TrainingDetails object. + + Args: + model_name (str, optional): existing model name, to be used to auto-discover training details. + training_job_name (str, optional): training job name, to be used to auto-discover training details. + objective_function (Dict[str, Any], optional): The objective function that is optimized during training (default: None). + training_observations (str, optional): Any observations about training (default: None). + training_job_details (Dict[str, Any], optional): Details about any associated training jobs (default: None). + sagemaker_session (Session): sagemaker session (default: None). + Returns: + sagemaker.model_card.TrainingDetails + """ + # if model_name was provided, auto-discover training details from model_overview + if model_name: + return TrainingDetails.from_model_overview( + model_overview=ModelOverview.from_model_name( + model_name=model_name, sagemaker_session=sagemaker_session + ), + sagemaker_session=sagemaker_session, + ) + # else, if training_job_name was provided, aut-discover training details from job name + elif training_job_name: + return TrainingDetails.from_training_job_name( + training_job_name=training_job_name, sagemaker_session=sagemaker_session + ) + # else create TrainingDetails using other provided params + else: + return TrainingDetails( + objective_function=cls.get_objective_function(**objective_function) if objective_function else None, + training_observations=training_observations, + training_job_details=cls.get_training_job_details(**training_job_details) + if training_job_details + else None, + ) + + @classmethod + @exception_handler + def get_evaluation_job( + cls, + name: str, + metric_file_s3_url: Optional[str] = None, + metric_type: Optional[Union[EvaluationMetricTypeEnum, str]] = None, + evaluation_observation: Optional[str] = None, + evaluation_job_arn: Optional[str] = None, + datasets: Optional[List[str]] = None, + metadata: Optional[dict] = None, + metric_groups: Optional[List[Dict[str, Any]]] = None, + ) -> EvaluationJob: + """Initialize an Evaluation Job object. + + Args: + name (str): The evaluation job name. + metric_file_s3_url (str. optional): metric file's s3 bucket url, to be used to auto-discover evaluation metrics (default: None). + metric_type (str, optional): one of model_card_metric_schema|clarify_bias|clarify_explainability|regression|binary_classification|multiclass_classification. + Required if metric_file_s3_url is provide (default: None). + evaluation_observation (str, optional): Any observations made during model evaluation (default: None). + evaluation_job_arn (str, optional): The Amazon Resource Name (ARN) of the evaluation job (default: None). + datasets (List[str], optional): Evaluation dataset locations. Maximum list length is 10 (default: None). + metadata (Optional[dict], optional): Additional attributes associated with the evaluation results (default: None). + metric_groups (List[Dict[str, Any], optional): An evaluation Metric Group object (default: None). + Returns: + sagemaker.model_card.EvaluationJob + """ + # define a map for metric_type + metric_type_map = { + "model_card_metric_schema": EvaluationMetricTypeEnum.MODEL_CARD_METRIC_SCHEMA, + "clarify_bias": EvaluationMetricTypeEnum.CLARIFY_BIAS, + "clarify_explainability": EvaluationMetricTypeEnum.CLARIFY_EXPLAINABILITY, + "regression": EvaluationMetricTypeEnum.REGRESSION, + "binary_classification": EvaluationMetricTypeEnum.BINARY_CLASSIFICATION, + "multiclass_classification": EvaluationMetricTypeEnum.MULTICLASS_CLASSIFICATION, + } + # if metric_file_s3_url provided, add metric group from json file in s3 + if metric_file_s3_url: + # check metric_type is provided + if not metric_type: + raise ValueError("metric_type type is required if evaluation metrics are to be loaded from s3 url") + if metric_type not in metric_type_map.keys(): + raise ValueError( + "metric_type must be one of model_card_metric_schema|clarify_bias|clarify_explainability|regression|binary_classification|multiclass_classification" + ) + # create EvaluationJob + evaluation_job = EvaluationJob(name=name) + # add metric group from s3 url + evaluation_job.add_metric_group_from_s3( + session=boto3.session.Session(), + s3_url=metric_file_s3_url, + metric_type=metric_type_map.get(metric_type), + ) + return evaluation_job + # else construct EvaluationJob from user provided params + else: + return EvaluationJob( + name=name, + evaluation_observation=evaluation_observation, + evaluation_job_arn=evaluation_job_arn, + datasets=datasets, + metadata=metadata, + metric_groups=[cls.get_metric_group(**metric_group) for metric_group in metric_groups] + if metric_groups + else None, + ) + + @classmethod + @exception_handler + def get_additional_information( + cls, + ethical_considerations: Optional[str] = None, + caveats_and_recommendations: Optional[str] = None, + custom_details: Optional[dict] = None, + ) -> AdditionalInformation: + """Initialize an Additional Information object. + + Args: + ethical_considerations (str, optional): Any ethical considerations to document about the model (default: None). + caveats_and_recommendations (str, optional): Caveats and recommendations for those who might use this model in their applications (default: None). + custom_details (dict, optional): Any additional custom information to document about the model (default: None). + Returns: + sagemaker.model_card.AdditionalInformation + """ + return AdditionalInformation( + ethical_considerations=ethical_considerations, + caveats_and_recommendations=caveats_and_recommendations, + custom_details=custom_details, + ) + + +class SolutionModelCard: + def __init__( + self, # NOSONAR:S107 this function is designed to take many arguments + name: str, + status: Optional[Union[ModelCardStatusEnum, str]] = ModelCardStatusEnum.DRAFT, + arn: Optional[str] = None, + version: Optional[int] = None, + created_by: Optional[dict] = None, + last_modified_by: Optional[dict] = None, + model_overview: Optional[ModelOverview] = None, + intended_uses: Optional[IntendedUses] = None, + training_details: Optional[TrainingDetails] = None, + evaluation_details: Optional[List[EvaluationJob]] = None, + additional_information: Optional[AdditionalInformation] = None, + sagemaker_session: Optional[Session] = None, + ): + self.name = name + self.arn = arn + self.status = status + self.version = version + self.created_by = created_by + self.last_modified_by = last_modified_by + self.model_overview = model_overview + self.intended_uses = intended_uses + self.training_details = training_details + self.evaluation_details = evaluation_details + self.additional_information = additional_information + self.sagemaker_session = sagemaker_session + + @exception_handler + def create_model_card(self): + # constructing ModelCard object + model_card = ModelCard( + name=self.name, + status=self.status, + arn=self.arn, + version=self.version, + created_by=self.created_by, + last_modified_by=self.last_modified_by, + model_overview=self.model_overview, + intended_uses=self.intended_uses, + training_details=self.training_details, + evaluation_details=self.evaluation_details, + additional_information=self.additional_information, + sagemaker_session=self.sagemaker_session, + ) + # create model card + model_card.create() + logger.info(f"Model Card with the name: {self.name} has been created...") + + @exception_handler + def delete_model_card(self): + # create ModelCard object for the card to be deleted + model_card = ModelCard(name=self.name) + # delete the model card + model_card.delete() + logger.info(f"Model Card with the name: {self.name} has been deleted...") + + @exception_handler + def describe_model_card(self) -> Dict[str, Any]: + model_card_info = self.sagemaker_session.sagemaker_client.describe_model_card(ModelCardName=self.name) + return model_card_info + + @classmethod + @exception_handler + def load_model_card(cls, name: str, version: Optional[int] = None, sagemaker_session: Session = None) -> ModelCard: + model_card = ModelCard.load( + name=name, + version=version, + sagemaker_session=sagemaker_session, + ) + logger.info(f"Model Card with the name: {name} has been loaded...") + return model_card + + @exception_handler + def update_model_card(self): + # load model card + model_card = self.load_model_card( + name=self.name, version=self.version, sagemaker_session=self.sagemaker_session + ) + # only include provided params (non-None vales), and exclude name, arn, and sagemaker_session + keywords = dict( + filter( + lambda elem: elem[1] is not None and elem[0] not in ["name", "arn", "sagemaker_session"], + self.__dict__.items(), + ) + ) + # update model card + model_card.update(**keywords) + logger.info(f"Model Card with the name: {self.name} has been updated...") + + @staticmethod + @exception_handler + def export_model_card( + model_card: ModelCard, + s3_output_path: str, + export_job_name: Optional[str] = None, + model_card_version: Optional[int] = None, + ): + + model_card.export_pdf( + s3_output_path=s3_output_path, export_job_name=export_job_name, model_card_version=model_card_version + ) + logger.info(f"Model Card with the name: {model_card.name} has been exported...") + + +class SolutionModelCardAPIs: + def __init__(self, event: Dict[str, Any], sagemaker_session: Session): + self.event = event + self.name = event.get("name") + self.status = event.get("status", "Draft") + self.arn = event.get("arn") + self.version = event.get("version") + self.created_by = event.get("created_by") + self.last_modified_by = event.get("last_modified_by") + self.expected_model_overview_params = [ + "model_id", + "model_name", + "model_description", + "model_version", + "problem_type", + "algorithm_type", + "model_creator", + "model_owner", + "model_artifact", + "inference_environment", + ] + self.expected_intended_uses_params = [ + "purpose_of_model", + "intended_uses", + "factors_affecting_model_efficiency", + "risk_rating", + "explanations_for_risk_rating", + ] + self.expected_training_details_params = [ + "model_name", + "training_job_name", + "objective_function", + "training_observations", + "training_job_details", + ] + + self.expected_evaluation_details_params = [ + "name", + "metric_file_s3_url", + "metric_type", + "evaluation_observation", + "evaluation_job_arn", + "datasets", + "metadata", + "metric_groups", + ] + self.expected_additional_information_params = [ + "ethical_considerations", + "caveats_and_recommendations", + "custom_details", + ] + self.model_overview = self._filter_expected_params( + event.get("model_overview", {}), self.expected_model_overview_params + ) + self.intended_uses = self._filter_expected_params( + event.get("intended_uses", {}), self.expected_intended_uses_params + ) + self.training_details = self._filter_expected_params( + event.get("training_details", {}), self.expected_training_details_params + ) + self.evaluation_details = [ + self._filter_expected_params(evaluation_details, self.expected_evaluation_details_params) + for evaluation_details in event.get("evaluation_details", []) + ] + self.additional_information = self._filter_expected_params( + event.get("additional_information", {}), self.expected_additional_information_params + ) + self.sagemaker_session = sagemaker_session + + @classmethod + @exception_handler + def _filter_expected_params(cls, params: Dict[str, Any], expected_params: List[str]) -> Dict[str, Any]: + return {key: params[key] for key in params.keys() if key in expected_params} + + @classmethod + @exception_handler + def _api_response(cls, message: str): + return { + "statusCode": 200, + "isBase64Encoded": False, + "body": json.dumps( + { + "message": message, + }, + indent=4, + cls=DateTimeEncoder, + ), + "headers": {"Content-Type": "plain/text"}, + } + + @exception_handler + def _create_solutions_card_object(self): + # create solution model card object + solutions_card_object = SolutionModelCard( + name=self.name, + status=self.status, + arn=self.arn, + version=self.version, + created_by=self.created_by, + last_modified_by=self.last_modified_by, + # create objects if some params are provided. Otherwise, pass None + model_overview=SolutionModelCardHelpers.get_model_overview(**self.model_overview) + if self.model_overview + else None, + intended_uses=SolutionModelCardHelpers.get_intended_uses(**self.intended_uses) + if self.intended_uses + else None, + training_details=SolutionModelCardHelpers.get_training_details(**self.training_details) + if self.training_details + else None, + evaluation_details=[ + SolutionModelCardHelpers.get_evaluation_job(**evaluation_details) + for evaluation_details in self.evaluation_details + ] + if self.evaluation_details + else None, + additional_information=SolutionModelCardHelpers.get_additional_information(**self.additional_information) + if self.additional_information + else None, + sagemaker_session=self.sagemaker_session, + ) + return solutions_card_object + + @exception_handler + def create(self): + # create model card + self._create_solutions_card_object().create_model_card() + # describe the create model card to return in the API call's response + return self._api_response( + SolutionModelCard(name=self.name, sagemaker_session=self.sagemaker_session).describe_model_card() + ) + + @exception_handler + def update(self): + # update model card + self._create_solutions_card_object().update_model_card() + # describe the updated model card to return in the API call's response + return self._api_response( + SolutionModelCard(name=self.name, sagemaker_session=self.sagemaker_session).describe_model_card() + ) + + @exception_handler + def delete(self): + SolutionModelCard(name=self.name, sagemaker_session=self.sagemaker_session).delete_model_card() + return self._api_response(f"Model card with the name: {self.name} has been deleted") + + @exception_handler + def describe(self): + # describe the model card + response = SolutionModelCard(name=self.name, sagemaker_session=self.sagemaker_session).describe_model_card() + logger.info(response) + return self._api_response(response) + + @exception_handler + def list_model_cards(self): + response = SolutionModelCardHelpers.list_model_cards(sagemaker_session=self.sagemaker_session) + logger.info(response) + return self._api_response(response) + + @exception_handler + def export_to_pdf(self, s3_output_path: str): + # load model card first + model_card = SolutionModelCard.load_model_card(name=self.name, sagemaker_session=self.sagemaker_session) + # export model card to s3 as a pdf file + SolutionModelCard.export_model_card( + model_card=model_card, + s3_output_path=s3_output_path, + export_job_name=self.event.get("export_job_name"), + model_card_version=int(self.event.get("model_card_version")) + if self.event.get("model_card_version") + else None, + ) + return self._api_response(f"Model card with the name: {self.name} has been exported to {s3_output_path}") diff --git a/source/lambdas/pipeline_orchestration/tests/test_pipeline_orchestration.py b/source/lambdas/pipeline_orchestration/tests/test_pipeline_orchestration.py index 410b85f..dc22a7c 100644 --- a/source/lambdas/pipeline_orchestration/tests/test_pipeline_orchestration.py +++ b/source/lambdas/pipeline_orchestration/tests/test_pipeline_orchestration.py @@ -50,6 +50,7 @@ update_stack, pipeline_status, DateTimeEncoder, + provision_model_card, ) from shared.wrappers import BadRequest from tests.fixtures.orchestrator_fixtures import ( @@ -98,6 +99,15 @@ def test_handler(): + # event["path"] == "/provisionpipeline" and pipeline_type is model_crad operation + with patch("pipeline_orchestration.index.provision_model_card") as mock_provision_card: + event = { + "httpMethod": "POST", + "path": "/provisionpipeline", + "body": json.dumps({"pipeline_type": "create_model_card", "test": "test"}), + } + handler(event, {}) + assert mock_provision_card.called is True # event["path"] == "/provisionpipeline" with patch("pipeline_orchestration.index.provision_pipeline") as mock_provision_pipeline: event = { @@ -519,6 +529,56 @@ def test_get_stack_name( ) +@patch("sagemaker.Session") +@patch("solution_model_card.SolutionModelCardAPIs.list_model_cards") +@patch("solution_model_card.SolutionModelCardAPIs.export_to_pdf") +@patch("solution_model_card.SolutionModelCardAPIs.delete") +@patch("solution_model_card.SolutionModelCardAPIs.describe") +@patch("solution_model_card.SolutionModelCardAPIs.update") +@patch("solution_model_card.SolutionModelCardAPIs.create") +def test_provision_model_card( + patched_create, patched_update, patched_describe, patched_delete, patched_export, patched_list, patched_session +): + # assert the create APIs is called when pipeline_type=create_model_card + event = dict(pipeline_type="create_model_card") + provision_model_card(event, patched_session) + assert patched_create.called is True + + # assert the create APIs is called when pipeline_type=update_model_card + event["pipeline_type"] = "update_model_card" + provision_model_card(event, patched_session) + assert patched_update.called is True + + # assert the create APIs is called when pipeline_type=delete_model_card + event["pipeline_type"] = "delete_model_card" + provision_model_card(event, patched_session) + assert patched_delete.called is True + + # assert the create APIs is called when pipeline_type=describe_model_card + event["pipeline_type"] = "describe_model_card" + provision_model_card(event, patched_session) + assert patched_describe.called is True + + # assert the create APIs is called when pipeline_type=export_model_card + event["pipeline_type"] = "export_model_card" + provision_model_card(event, patched_session) + assert patched_export.called is True + + # assert the create APIs is called when pipeline_type=list_model_cardd + event["pipeline_type"] = "list_model_cards" + provision_model_card(event, patched_session) + assert patched_list.called is True + + # assert for error if the pipeline_type is incorrect + event["pipeline_type"] = "wrong_pipeline_type" + with pytest.raises(BadRequest) as error_info: + provision_model_card(event, patched_session) + assert ( + str(error_info.value) + == "pipeline_type must be on of create_model_card|update_model_card|describe_model_card|delete_model_card|export_model_card" + ) + + def test_get_required_keys( api_byom_event, # NOSONAR:S107 this test function is designed to take many fixtures api_data_quality_event, diff --git a/source/lambdas/pipeline_orchestration/tests/test_solution_model_card.py b/source/lambdas/pipeline_orchestration/tests/test_solution_model_card.py new file mode 100644 index 0000000..0464d8b --- /dev/null +++ b/source/lambdas/pipeline_orchestration/tests/test_solution_model_card.py @@ -0,0 +1,365 @@ +import unittest +import pytest +import json +from unittest import TestCase +from unittest.mock import patch, Mock +from sagemaker.model_card import ( + Environment, + ModelOverview, + RiskRatingEnum, + TrainingJobDetails, + TrainingDetails, + ModelCard, +) +from solution_model_card import SolutionModelCardHelpers, SolutionModelCard, SolutionModelCardAPIs + + +class TestSolutionModelCardHelpers(unittest.TestCase): + def setUp(self): + self.model_name = "test-model" + self.model_id = "arn:model-name" + self.container_image = [".dkr.ecr.us-east-2.amazonaws.com/sagemaker-sklearn-automl:2.5-1-cpu-py3"] + # intended uses + self.intended_uses = dict( + purpose_of_model="customer churn", + intended_uses="marketing allocation", + factors_affecting_model_efficiency="data quality", + risk_rating=RiskRatingEnum.MEDIUM, + ) + # objective function + self.objective_function = dict(function="Maximize", facet="AUC") + # metric + self.metric = dict(name="metric", type="linear_graph", value=[1.4, 1.5, 1.6]) + + # metric group + self._metric_group_name = "metric-group" + # additional information + self.additional_information = dict( + ethical_considerations="evaluate model regularly", custom_details={"department": "marketing"} + ) + + # training job details + self.training_job_details = dict( + training_arn="training-job-arn", + training_datasets=["s3://test-bucket/data/csv"], + training_environment=self.container_image, + ) + + # training details + self.training_details = dict( + training_arn="training-jon-arn", + training_metrics=[ + SolutionModelCardHelpers.get_training_metric(name=self.metric["name"], value=self.metric["value"]) + ], + ) + self.training_job_name = "test-job-name" + self.list_model_cards_response = { + "ModelCardSummaries": [ + { + "ModelCardName": "TestModelCard", + "ModelCardArn": "arn:aws:sagemaker::account-id:model-card/TestModelCard", + "ModelCardStatus": "Draft", + } + ] + } + + def test_get_environment(self): + environment = SolutionModelCardHelpers.get_environment(self.container_image) + assert environment.container_image == self.container_image + + @patch("sagemaker.model_card.ModelOverview.from_model_name") + def test_get_model_overview(self, patched_model_overview): + patched_model_overview.return_value = ModelOverview( + model_name=self.model_name, + model_id=self.model_id, + inference_environment=Environment(container_image=self.container_image), + ) + model_overview = SolutionModelCardHelpers.get_model_overview(model_name=self.model_name) + assert model_overview.model_name == self.model_name + assert model_overview.model_id == self.model_id + assert model_overview.inference_environment.container_image == self.container_image + + def test_get_intended_uses(self): + intended_uses = SolutionModelCardHelpers.get_intended_uses(**self.intended_uses) + assert intended_uses.purpose_of_model == self.intended_uses["purpose_of_model"] + assert intended_uses.risk_rating == RiskRatingEnum.MEDIUM + assert intended_uses.explanations_for_risk_rating is None + + def test_get_objective_function(self): + objective_function = SolutionModelCardHelpers.get_objective_function(**self.objective_function) + assert objective_function.function.function == self.objective_function["function"] + assert objective_function.function.facet == self.objective_function["facet"] + assert objective_function.function.condition is None + assert objective_function.notes is None + + def test_get_metric(self): + metric = SolutionModelCardHelpers.get_metric(**self.metric) + assert metric.name == self.metric["name"] + assert metric.type == self.metric["type"] + TestCase().assertEqual(metric.value, self.metric["value"]) + assert metric.notes is None + assert metric.x_axis_name is None + + def test_get_metric_group(self): + metric_group = SolutionModelCardHelpers.get_metric_group( + name=self._metric_group_name, + ) + assert metric_group.name == self._metric_group_name + assert len(metric_group.metric_data) == 0 + # add metric + metric_group.add_metric(SolutionModelCardHelpers.get_metric(**self.metric)) + assert metric_group.metric_data[0].name == self.metric["name"] + + def test_get_training_metric(self): + training_metric = SolutionModelCardHelpers.get_training_metric( + name=self.metric["name"], value=self.metric["value"] + ) + assert training_metric.name == self.metric["name"] + assert training_metric.value == self.metric["value"] + assert training_metric.notes is None + + def test_get_training_job_details(self): + training_job_details = SolutionModelCardHelpers.get_training_job_details(**self.training_job_details) + assert training_job_details.training_arn == self.training_job_details["training_arn"] + assert len(training_job_details.user_provided_training_metrics) == 0 + + @patch("sagemaker.model_card.TrainingDetails.from_training_job_name") + @patch("sagemaker.model_card.TrainingDetails.from_model_overview") + @patch("sagemaker.model_card.ModelOverview.from_model_name") + def test_get_training_details( + self, patched_from_model_name, patched_from_model_overview, patched_from_training_job_name + ): + # create the return value + returned_training_details = TrainingDetails(training_job_details=TrainingJobDetails(**self.training_details)) + # 1 - assert when model_name is provided + # set the return vale for ModelOverview.from_model_name + patched_from_model_name.return_value = ModelOverview( + model_name=self.model_name, + model_id=self.model_id, + inference_environment=Environment(container_image=self.container_image), + ) + # set the return vale for TrainingDetails.from_model_overview + patched_from_model_overview.return_value = returned_training_details + training_details_from_model_overview = SolutionModelCardHelpers.get_training_details(model_name=self.model_name) + assert ( + training_details_from_model_overview.training_job_details.training_arn + == self.training_details["training_arn"] + ) + TestCase().assertEqual( + training_details_from_model_overview.training_job_details.training_metrics, + self.training_details["training_metrics"], + ) + # 2 - assert when training_job_name is provided + patched_from_training_job_name.return_value = returned_training_details + training_details_from_job_name = SolutionModelCardHelpers.get_training_details( + training_job_name=self.training_job_name + ) + assert training_details_from_job_name.training_job_details.training_arn == self.training_details["training_arn"] + + # 3 - assert when other params are provided + training_details = SolutionModelCardHelpers.get_training_details(training_job_details=self.training_job_details) + + assert training_details.training_job_details.training_arn == self.training_job_details["training_arn"] + assert len(training_details.training_job_details.user_provided_training_metrics) == 0 + assert training_details.training_observations is None + + def test_get_additional_information(self): + additional_information = SolutionModelCardHelpers.get_additional_information(**self.additional_information) + assert additional_information.ethical_considerations == self.additional_information["ethical_considerations"] + assert additional_information.caveats_and_recommendations is None + TestCase().assertEqual(additional_information.custom_details, self.additional_information["custom_details"]) + + @patch("sagemaker.Session") + def test_list_model_cards(self, patched_session): + patched_session.sagemaker_client.list_model_cards = Mock(return_value=self.list_model_cards_response) + response = SolutionModelCardHelpers.list_model_cards(patched_session) + assert response["ModelCardSummaries"][0]["ModelCardName"] == "TestModelCard" + + @patch("sagemaker.model_card.EvaluationJob.add_metric_group_from_s3") + def test_get_evaluation_job(self, patched_add_metric_group_from_s3): + job_name = "test-job" + evaluation_job_arn = f"arn:evaluation-job/{job_name}" + metric_file_s3_url = "s3://bucket/clarify.json" + # test for error when metric_file_s3_url but metric_type is not provided + with pytest.raises(ValueError) as error_info: + SolutionModelCardHelpers.get_evaluation_job(name=job_name, metric_file_s3_url=metric_file_s3_url) + assert ( + str(error_info.value) == "metric_type type is required if evaluation metrics are to be loaded from s3 url" + ) + # test for error when metric_file_s3_url but metric_type has a wrong value + with pytest.raises(ValueError) as error_info: + SolutionModelCardHelpers.get_evaluation_job( + name=job_name, metric_file_s3_url=metric_file_s3_url, metric_type="wrong-value" + ) + assert ( + str(error_info.value) + == "metric_type must be one of model_card_metric_schema|clarify_bias|clarify_explainability|regression|binary_classification|multiclass_classification" + ) + # test when metric_file_s3_url and metric_type are provided + evaluation_job = SolutionModelCardHelpers.get_evaluation_job( + name=job_name, metric_file_s3_url=metric_file_s3_url, metric_type="clarify_bias" + ) + assert evaluation_job.name == job_name + + # test with metric_file_s3_url and metric_type not provided + evaluation_job = SolutionModelCardHelpers.get_evaluation_job( + name=job_name, evaluation_job_arn=evaluation_job_arn + ) + assert evaluation_job.evaluation_job_arn == evaluation_job_arn + + def test_get_additional_information(self): + ethical_considerations = "no ethical concerns" + caveats_and_recommendations = "some recommendations" + additional_information = SolutionModelCardHelpers.get_additional_information( + ethical_considerations=ethical_considerations, caveats_and_recommendations=caveats_and_recommendations + ) + assert additional_information.ethical_considerations == ethical_considerations + assert additional_information.caveats_and_recommendations == caveats_and_recommendations + assert additional_information.custom_details is None + + +class TestSolutionModelCard(unittest.TestCase): + def setUp(self): + self.name = "test-card" + self.status = "Draft" + self.example_card = ModelCard(name=self.name, status=self.status) + self.s3_output_path = "s3://bucket/exports" + + @patch("sagemaker.model_card.ModelCard.create") + def test_create_model_card(self, patched_card_create): + SolutionModelCard(name=self.name, status=self.status).create_model_card() + assert patched_card_create.called is True + + @patch("sagemaker.model_card.ModelCard.delete") + def test_delete_model_card(self, patched_card_delete): + SolutionModelCard(name=self.name).delete_model_card() + assert patched_card_delete.called is True + + @patch("sagemaker.Session") + def test_describe_model_cards(self, patched_session): + patched_session.sagemaker_client.describe_model_card = Mock( + return_value={"name": self.name, "status": self.status} + ) + response = SolutionModelCard(name=self.name, sagemaker_session=patched_session).describe_model_card() + assert response["name"] == self.name + assert response["status"] == self.status + + @patch("sagemaker.model_card.ModelCard.load") + def test_load_model_card(self, patched_card_load): + patched_card_load.return_value = self.example_card + response = SolutionModelCard.load_model_card(name=self.name) + assert response.name == self.name + assert response.status == self.status + + @patch("sagemaker.model_card.ModelCard.update") + @patch("solution_model_card.SolutionModelCard.load_model_card") + def test_update_model_card(self, patched_load_model_card, patched_card_update): + patched_load_model_card.return_value = self.example_card + card = SolutionModelCard(name=self.name, status="Approved") + card.update_model_card() + patched_card_update.assert_called_with(status="Approved") + + @patch("sagemaker.model_card.ModelCard.export_pdf") + @patch("solution_model_card.SolutionModelCard.load_model_card") + def test_export_model_card(self, patched_load_model_card, patched_card_export_pdf): + patched_load_model_card.return_value = self.example_card + card = SolutionModelCard.load_model_card(name=self.name) + SolutionModelCard.export_model_card(model_card=card, s3_output_path=self.s3_output_path) + patched_card_export_pdf.assert_called_with( + s3_output_path=self.s3_output_path, export_job_name=None, model_card_version=None + ) + + +class TestSolutionModelCardAPIs(unittest.TestCase): + def setUp(self): + self.event = dict( + name="test-card", + version=1, + created_by="DS", + model_overview=dict(model_description="model-description", unexpected="random-value"), + ) + self.s3_url = "s3://test-bucket" + self.message = "model card has been deleted" + self.api_response = { + "statusCode": 200, + "isBase64Encoded": False, + "body": json.dumps( + { + "message": self.message, + }, + indent=4, + ), + "headers": {"Content-Type": "plain/text"}, + } + + @patch("sagemaker.Session") + def test__filter_expected_params(self, patched_session): + card_api = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + TestCase().assertEqual( + SolutionModelCardAPIs._filter_expected_params( + self.event["model_overview"], expected_params=card_api.expected_model_overview_params + ), + {"model_description": "model-description"}, + ) + + def test__api_response(self): + TestCase().assertEqual(SolutionModelCardAPIs._api_response(message=self.message), self.api_response) + + @patch("sagemaker.Session") + def test__create_solutions_card_object(self, patched_session): + solutions_card_object = SolutionModelCardAPIs( + event=self.event, sagemaker_session=patched_session + )._create_solutions_card_object() + assert solutions_card_object.name == self.event["name"] + assert ( + solutions_card_object.model_overview.model_description == self.event["model_overview"]["model_description"] + ) + assert solutions_card_object.additional_information is None + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCard.describe_model_card") + @patch("solution_model_card.SolutionModelCard.create_model_card") + def test_create(self, patched_create_model_card, patched_describe_model_card, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.create() + assert patched_create_model_card.called is True + assert patched_describe_model_card.called is True + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCard.describe_model_card") + @patch("solution_model_card.SolutionModelCard.update_model_card") + def test_update(self, patched_update_model_card, patched_describe_model_card, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.update() + assert patched_update_model_card.called is True + assert patched_describe_model_card.called is True + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCard.delete_model_card") + def test_delete(self, patched_delete_model_card, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.delete() + assert patched_delete_model_card.called is True + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCard.describe_model_card") + def test_describe(self, patched_describe_model_card, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.describe() + assert patched_describe_model_card.called is True + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCardHelpers.list_model_cards") + def test_list_model_cards(self, patched_list_model_cards, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.list_model_cards() + assert patched_list_model_cards.called is True + + @patch("sagemaker.Session") + @patch("solution_model_card.SolutionModelCard.export_model_card") + @patch("solution_model_card.SolutionModelCard.load_model_card") + def test_export_to_pdf(self, patched_load_model_card, patched_export_model_card, patched_session): + solutions_card_object = SolutionModelCardAPIs(event=self.event, sagemaker_session=patched_session) + solutions_card_object.export_to_pdf(s3_output_path=self.s3_url) + assert patched_load_model_card.called is True + assert patched_export_model_card.called is True diff --git a/source/lib/blueprints/byom/autopilot_training_pipeline.py b/source/lib/blueprints/byom/autopilot_training_pipeline.py index fff3e66..3eca8ec 100644 --- a/source/lib/blueprints/byom/autopilot_training_pipeline.py +++ b/source/lib/blueprints/byom/autopilot_training_pipeline.py @@ -172,7 +172,6 @@ def _create_job_notification_rules(self): eventbridge_rule_to_sns( scope=self, logical_id="AutopilotJobTunerRule", - rule_name="autopilot_job_notification_rule", description="EventBridge rule to notify the admin on the status change of the hyperparameter job used by the autopilot job", source=event_source, detail_type=["SageMaker HyperParameter Tuning Job State Change"], @@ -196,7 +195,6 @@ def _create_job_notification_rules(self): eventbridge_rule_to_sns( scope=self, logical_id="AutopilotJobProcessingRule", - rule_name="autopilot_job_processing_rule", description="EventBridge rule to notify the admin on the status change of the last two processing jobs used the autopilot job", source=event_source, detail_type=["SageMaker Processing Job State Change"], @@ -223,7 +221,6 @@ def _create_job_notification_rules(self): eventbridge_rule_to_sns( scope=self, logical_id="AutopilotJobInterProcessingRule", - rule_name="autopilot_job_inter_processing_rule", description="EventBridge rule to notify the admin on the status change of the intermidate processing jobs used the autopilot job", source=event_source, detail_type=["SageMaker Processing Job State Change"], @@ -250,7 +247,6 @@ def _create_job_notification_rules(self): eventbridge_rule_to_sns( scope=self, logical_id="AutopilotJobTrainingRule", - rule_name="autopilot_job_training_rule", description="EventBridge rule to notify the admin on the status change of the intermidate training jobs used the autopilot job", source=event_source, detail_type=["SageMaker Training Job State Change"], @@ -276,7 +272,6 @@ def _create_job_notification_rules(self): eventbridge_rule_to_sns( scope=self, logical_id="AutopilotJobTransformRule", - rule_name="autopilot_job_transform_rule", description="EventBridge rule to notify the admin on the status change of the intermidate transform jobs used the autopilot job", source=event_source, detail_type=["SageMaker Transform Job State Change"], diff --git a/source/lib/blueprints/byom/lambdas/batch_transform/main.py b/source/lib/blueprints/byom/lambdas/batch_transform/main.py index 76b3985..d8515ee 100644 --- a/source/lib/blueprints/byom/lambdas/batch_transform/main.py +++ b/source/lib/blueprints/byom/lambdas/batch_transform/main.py @@ -19,7 +19,7 @@ sm_client = get_client("sagemaker") -def handler(event, context): +def handler(*_): try: model_name = os.environ.get("model_name").lower() batch_inference_data = os.environ.get("batch_inference_data") diff --git a/source/lib/blueprints/byom/lambdas/create_baseline_job/baselines_helper.py b/source/lib/blueprints/byom/lambdas/create_baseline_job/baselines_helper.py index db51a57..cca69fd 100644 --- a/source/lib/blueprints/byom/lambdas/create_baseline_job/baselines_helper.py +++ b/source/lib/blueprints/byom/lambdas/create_baseline_job/baselines_helper.py @@ -12,6 +12,7 @@ # ##################################################################################################################### from typing import Any, Dict, List, Optional, Union import logging +import json import sagemaker from botocore.client import BaseClient from sagemaker.model_monitor import DefaultModelMonitor @@ -414,6 +415,15 @@ def _create_model_bias_baseline( **model_bias_baseline_job_args["suggest_args"], ) + # get the analysis_config.json for Model Bias monitor + analysis_config = model_bias_monitor.latest_baselining_job_config.analysis_config._to_dict() + logger.info(f"Model Bias analysis config: {json.dumps(analysis_config)}") + + # upload ModelBias analysis_config.json file to S3 + self._upload_analysis_config( + output_s3_uri=f"{self.output_s3_uri}/monitor/analysis_config.json", analysis_config=analysis_config + ) + return model_bias_baseline_job @exception_handler @@ -444,6 +454,15 @@ def _create_model_explainability_baseline( **model_explainability_baseline_job_args["suggest_args"], ) + # get the analysis_config.json for Explainability monitor + analysis_config = model_explainability_monitor.latest_baselining_job_config.analysis_config._to_dict() + logger.info(f"model_explainability analysis config: {json.dumps(analysis_config)}") + + # upload Explainability analysis_config.json file to S3 + self._upload_analysis_config( + output_s3_uri=f"{self.output_s3_uri}/monitor/analysis_config.json", analysis_config=analysis_config + ) + return model_explainability_baseline_job def _is_valid_argument_value(self, value: str) -> bool: @@ -494,3 +513,12 @@ def get_baseline_dataset_header(bucket_name: str, file_key: str, s3_client: Base header = dataset.split("\n")[0].split(",") return header + + @exception_handler + def _upload_analysis_config(self, output_s3_uri: str, analysis_config: Dict[str, Any]): + logger.info(f"uploading analysis_confg.json to {output_s3_uri}") + analysis_config_uri = sagemaker.s3.S3Uploader.upload_string_as_file_body( + json.dumps(analysis_config), desired_s3_uri=output_s3_uri, sagemaker_session=self.sagemaker_session + ) + + logger.info(f"analysis_confg.json uri is: {analysis_config_uri}") diff --git a/source/lib/blueprints/byom/lambdas/create_baseline_job/tests/test_create_data_baseline.py b/source/lib/blueprints/byom/lambdas/create_baseline_job/tests/test_create_data_baseline.py index 3c11245..ad771e8 100644 --- a/source/lib/blueprints/byom/lambdas/create_baseline_job/tests/test_create_data_baseline.py +++ b/source/lib/blueprints/byom/lambdas/create_baseline_job/tests/test_create_data_baseline.py @@ -10,7 +10,7 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### -from unittest.mock import patch +from unittest.mock import patch, PropertyMock from unittest import TestCase import pytest import boto3 @@ -228,22 +228,82 @@ def test_create_model_quality_baseline(mocked_model_monitor_suggest_baseline, mo mocked_model_monitor_suggest_baseline.assert_called_with(**expected_baseline_args["suggest_args"]) +class AnalysisConfigMock: + def _to_dict(self): + return { + "methods": { + "shap": { + "baseline": [ + [ + 93, + 0, + 146, + 102, + 164, + ] + ], + "num_samples": 100, + "agg_method": "mean_abs", + "use_logit": False, + "save_local_shap_values": True, + } + }, + "predictor": { + "model_name": "MLOpsSagemakerModel-Ku442sVZS7ER", + "instance_type": "ml.m5.large", + "initial_instance_count": 1, + "accept_type": "text/csv", + }, + "headers": [ + "Account Length", + "VMail Message", + "Day Mins", + "Day Calls", + ], + } + + +class BaselineJobConfigMock: + analysis_config = AnalysisConfigMock() + + +@patch("baselines_helper.SolutionSageMakerBaselines._upload_analysis_config") +@patch( + "baselines_helper.ModelBiasMonitor.latest_baselining_job_config", + create=True, + new_callable=PropertyMock, + return_value=BaselineJobConfigMock, +) @patch("baselines_helper.ModelBiasMonitor.suggest_baseline") -def test_create_model_bias_baseline(mocked_model_bias_suggest_baseline, mocked_sagemaker_baselines_instance): +def test_create_model_bias_baseline( + mocked_model_bias_suggest_baseline, mocked_analysis, mocked_upload, mocked_sagemaker_baselines_instance +): sagemaker_baselines = mocked_sagemaker_baselines_instance("ModelBias") expected_baseline_args = sagemaker_baselines._get_baseline_job_args() sagemaker_baselines._create_model_bias_baseline(expected_baseline_args) mocked_model_bias_suggest_baseline.assert_called_with(**expected_baseline_args["suggest_args"]) + mocked_analysis.assert_called() +@patch("baselines_helper.SolutionSageMakerBaselines._upload_analysis_config") +@patch( + "baselines_helper.ModelExplainabilityMonitor.latest_baselining_job_config", + create=True, + new_callable=PropertyMock, + return_value=BaselineJobConfigMock, +) @patch("baselines_helper.ModelExplainabilityMonitor.suggest_baseline") def test_create_model_explainability_baseline( - mocked_model_explainability_suggest_baseline, mocked_sagemaker_baselines_instance + mocked_model_explainability_suggest_baseline, + mocked_analysis, + mocked_upload, + mocked_sagemaker_baselines_instance, ): sagemaker_baselines = mocked_sagemaker_baselines_instance("ModelExplainability") expected_baseline_args = sagemaker_baselines._get_baseline_job_args() sagemaker_baselines._create_model_explainability_baseline(expected_baseline_args) mocked_model_explainability_suggest_baseline.assert_called_with(**expected_baseline_args["suggest_args"]) + mocked_analysis.assert_called() def test_get_baseline_dataset_header(mocked_baseline_dataset_header): diff --git a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/main.py b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/main.py index 7ae467c..414df8b 100644 --- a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/main.py +++ b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/main.py @@ -32,7 +32,7 @@ cp_client = get_client("codepipeline") -def lambda_handler(event, context): +def lambda_handler(event, _): """The Lambda function handler If a continuing job then checks the CloudFormation stackset and its instances status diff --git a/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt b/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt index 0718cb4..550b3ec 100644 --- a/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt +++ b/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt @@ -1 +1,4 @@ -sagemaker==2.39.0 \ No newline at end of file +botocore==1.29.20 +boto3==1.26.20 +awscli==1.27.20 +sagemaker==2.118.0 \ No newline at end of file diff --git a/source/lib/blueprints/byom/model_training_pipeline.py b/source/lib/blueprints/byom/model_training_pipeline.py index 45166df..049d0ce 100644 --- a/source/lib/blueprints/byom/model_training_pipeline.py +++ b/source/lib/blueprints/byom/model_training_pipeline.py @@ -236,7 +236,6 @@ def _create_job_notification_rule( eventbridge_rule_to_sns( scope=self, logical_id="JobNotificationRule", - rule_name="job_notification_rule", description="EventBridge rule to notify the admin on the status change of the job", source=["aws.sagemaker"], detail_type=values_map[self.training_type]["detail_type"], diff --git a/source/lib/blueprints/byom/multi_account_codepipeline.py b/source/lib/blueprints/byom/multi_account_codepipeline.py index a03fb5f..80f273b 100644 --- a/source/lib/blueprints/byom/multi_account_codepipeline.py +++ b/source/lib/blueprints/byom/multi_account_codepipeline.py @@ -103,7 +103,6 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: [dev_account_id.value_as_string], [dev_org_id.value_as_string], [core.Aws.REGION], - assets_bucket, f"{stack_name.value_as_string}-dev-{unique_id}", delegated_admin_account_condition, ) @@ -127,7 +126,6 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: [staging_account_id.value_as_string], [staging_org_id.value_as_string], [core.Aws.REGION], - assets_bucket, f"{stack_name.value_as_string}-staging-{unique_id}", delegated_admin_account_condition, ) @@ -151,7 +149,6 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: [prod_account_id.value_as_string], [prod_org_id.value_as_string], [core.Aws.REGION], - assets_bucket, f"{stack_name.value_as_string}-prod-{unique_id}", delegated_admin_account_condition, ) diff --git a/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py b/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py index 364c74f..033012e 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py +++ b/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py @@ -42,6 +42,7 @@ autopilot_job_policy, autopilot_job_endpoint_policy, training_job_policy, + sagemaker_tags_policy_statement, ) @@ -244,6 +245,9 @@ def create_baseline_job_lambda( # add conditions to KMS and ECR policies core.Aspects.of(kms_policy).add(ConditionalResources(kms_key_arn_provided_condition)) + # sagemaker tags permissions + sagemaker_tags_policy = sagemaker_tags_policy_statement() + # create sagemaker role sagemaker_role = create_service_role( scope, @@ -270,6 +274,7 @@ def create_baseline_job_lambda( sagemaker_logs_policy.attach_to_role(sagemaker_role) sagemaker_role.add_to_policy(create_baseline_job_policy) + sagemaker_role.add_to_policy(sagemaker_tags_policy) # add extra permissions for "ModelBias", "ModelExplainability" baselines if monitoring_type in ["ModelBias", "ModelExplainability"]: lambda_role.add_to_policy(baseline_lambda_get_model_name_policy(endpoint_name)) @@ -285,6 +290,7 @@ def create_baseline_job_lambda( ) ) lambda_role.add_to_policy(create_baseline_job_policy) + lambda_role.add_to_policy(sagemaker_tags_policy) lambda_role.add_to_policy(s3_write) lambda_role.add_to_policy(s3_read) add_logs_policy(lambda_role) @@ -355,7 +361,6 @@ def create_stackset_action( account_ids, org_ids, regions, - assets_bucket, stack_name, delegated_admin_condition, ): @@ -372,7 +377,6 @@ def create_stackset_action( :account_ids: list of AWS accounts where the stack with be deployed :org_ids: list of AWS organizational ids where the stack with be deployed :regions: list of regions where the stack with be deployed - :assets_bucket: the bucket cdk object where pipeline assets are stored :stack_name: name of the stack to be deployed :delegated_admin_condition: CDK condition to indicate if a delegated admin account is used :return: codepipeline invokeLambda action in a form of a CDK object that can be attached to a codepipeline stage @@ -446,12 +450,11 @@ def create_stackset_action( def create_cloudformation_action( - scope, action_name, stack_name, source_output, template_file, template_parameters_file, run_order=1 + action_name, stack_name, source_output, template_file, template_parameters_file, run_order=1 ): """ create_cloudformation_action a CloudFormation action to be added to AWS Codepipeline stage - :scope: CDK Construct scope that's needed to create CDK resources :action_name: name of the StackSet action :stack_name: name of the stack to be deployed :source_output: CDK object of the Source action's output @@ -902,9 +905,7 @@ def model_training_job( return training_lambda -def eventbridge_rule_to_sns( - scope, logical_id, rule_name, description, source, detail_type, detail, target_sns_topic, sns_message -): +def eventbridge_rule_to_sns(scope, logical_id, description, source, detail_type, detail, target_sns_topic, sns_message): event_rule = events.Rule( scope, logical_id, diff --git a/source/lib/blueprints/byom/pipeline_definitions/helpers.py b/source/lib/blueprints/byom/pipeline_definitions/helpers.py index f9f86bb..cdae713 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/helpers.py +++ b/source/lib/blueprints/byom/pipeline_definitions/helpers.py @@ -305,4 +305,4 @@ def suppress_delegated_admin_policy(): } ] } - } \ No newline at end of file + } diff --git a/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py b/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py index 92e93c2..e4504aa 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py +++ b/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py @@ -37,12 +37,12 @@ def sagemaker_policy_statement(is_realtime_pipeline, endpoint_name, endpoint_nam # extend actions actions.extend( [ - "sagemaker:CreateEndpointConfig", + "sagemaker:CreateEndpointConfig", # NOSONAR: permission needs to be repeated for clarity "sagemaker:DescribeEndpointConfig", # NOSONAR: permission needs to be repeated for clarity - "sagemaker:DeleteEndpointConfig", + "sagemaker:DeleteEndpointConfig", # NOSONAR: permission needs to be repeated for clarity "sagemaker:CreateEndpoint", # NOSONAR: permission needs to be repeated for clarity "sagemaker:DescribeEndpoint", # NOSONAR: permission needs to be repeated for clarity - "sagemaker:DeleteEndpoint", + "sagemaker:DeleteEndpoint", # NOSONAR: permission needs to be repeated for clarity ] ) @@ -180,11 +180,16 @@ def sagemaker_monitor_policy_statement(baseline_job_name, monitoring_schedule_na "sagemaker:DescribeModel", "sagemaker:DescribeEndpointConfig", "sagemaker:DescribeEndpoint", + "sagemaker:CreateEndpointConfig", + "sagemaker:CreateEndpoint", "sagemaker:CreateMonitoringSchedule", "sagemaker:DescribeMonitoringSchedule", "sagemaker:StopMonitoringSchedule", "sagemaker:DeleteMonitoringSchedule", "sagemaker:DescribeProcessingJob", + "sagemaker:DeleteEndpointConfig", + "sagemaker:DeleteEndpoint", + "sagemaker:InvokeEndpoint", ] # common resources resources = [ @@ -193,6 +198,8 @@ def sagemaker_monitor_policy_statement(baseline_job_name, monitoring_schedule_na f"{sagemaker_arn_prefix}:endpoint/{endpoint_name}", f"{sagemaker_arn_prefix}:monitoring-schedule/{monitoring_schedule_name}", f"{sagemaker_arn_prefix}:processing-job/{baseline_job_name}", + f"{sagemaker_arn_prefix}:endpoint-config/sm-clarify-config*", + f"{sagemaker_arn_prefix}:endpoint/sm-clarify-*", ] # create a map of monitoring type -> required permissions/resources @@ -481,11 +488,13 @@ def cloudformation_stackset_instances_policy(stack_name, account_id): "cloudformation:CreateStackInstances", "cloudformation:DeleteStackInstances", "cloudformation:UpdateStackSet", + "lambda:TagResource", ], resources=[ f"arn:{core.Aws.PARTITION}:cloudformation::{account_id}:stackset-target/{stack_name}:*", f"arn:{core.Aws.PARTITION}:cloudformation:{core.Aws.REGION}::type/resource/*", f"arn:{core.Aws.PARTITION}:cloudformation:{core.Aws.REGION}:{account_id}:stackset/{stack_name}:*", + f"arn:{core.Aws.PARTITION}:lambda:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:function:*", ], ) @@ -515,7 +524,7 @@ def create_orchestrator_policy( blueprint_repository_bucket, assets_s3_bucket_name, ): - return iam.Policy( + orchestrator_policy = iam.Policy( scope, "lambdaOrchestratorPolicy", statements=[ @@ -707,9 +716,40 @@ def create_orchestrator_policy( f"arn:{core.Aws.PARTITION}:events:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:event-bus/*", ], ), + # SageMaker Model Card permissions + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=[ + # to perform model card operations + "sagemaker:CreateModelCard", + "sagemaker:DescribeModelCard", + "sagemaker:UpdateModelCard", + "sagemaker:DeleteModelCard", + "sagemaker:CreateModelCardExportJob", + "sagemaker:DescribeModelCardExportJob", + "sagemaker:DescribeModel", + # to extract training details information + "sagemaker:DescribeTrainingJob", + ], + resources=[ + f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:model-card/*", + f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:model/*", + f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:training-job/*", + ], + ), + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=["sagemaker:ListModelCards", "sagemaker:Search"], + # ListModelCards/sagemaker:Search do not have a scoped-down resource + resources=[ + "*", + ], + ), ], ) + return orchestrator_policy + def create_invoke_lambda_policy(lambda_functions_list): return iam.PolicyStatement( diff --git a/source/lib/blueprints/byom/pipeline_definitions/sagemaker_model_monitor_construct.py b/source/lib/blueprints/byom/pipeline_definitions/sagemaker_model_monitor_construct.py index 76e111f..3f0a8b3 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/sagemaker_model_monitor_construct.py +++ b/source/lib/blueprints/byom/pipeline_definitions/sagemaker_model_monitor_construct.py @@ -299,7 +299,8 @@ def _create_model_bias_job_definition( self.scope, id, model_bias_app_specification=sagemaker.CfnModelBiasJobDefinition.ModelBiasAppSpecificationProperty( - config_uri=f"{self.baseline_job_output_location}/analysis_config.json", image_uri=self.image_uri + config_uri=f"{self.baseline_job_output_location}/monitor/analysis_config.json", + image_uri=self.image_uri, ), model_bias_baseline_config=sagemaker.CfnModelBiasJobDefinition.ModelBiasBaselineConfigProperty( constraints_resource=sagemaker.CfnModelBiasJobDefinition.ConstraintsResourceProperty( @@ -366,7 +367,7 @@ def _create_model_explainability_job_definition( self.scope, id, model_explainability_app_specification=sagemaker.CfnModelExplainabilityJobDefinition.ModelExplainabilityAppSpecificationProperty( - config_uri=f"{self.baseline_job_output_location}/analysis_config.json", image_uri=self.image_uri + config_uri=f"{self.baseline_job_output_location}/monitor/analysis_config.json", image_uri=self.image_uri ), model_explainability_baseline_config=sagemaker.CfnModelExplainabilityJobDefinition.ModelExplainabilityBaselineConfigProperty( constraints_resource=sagemaker.CfnModelExplainabilityJobDefinition.ConstraintsResourceProperty( diff --git a/source/lib/blueprints/byom/single_account_codepipeline.py b/source/lib/blueprints/byom/single_account_codepipeline.py index 611a1fe..20f24b1 100644 --- a/source/lib/blueprints/byom/single_account_codepipeline.py +++ b/source/lib/blueprints/byom/single_account_codepipeline.py @@ -56,7 +56,6 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: # create cloudformation action cloudformation_action = create_cloudformation_action( - self, "deploy_stack", stack_name.value_as_string, source_output, diff --git a/source/lib/mlops_orchestrator_stack.py b/source/lib/mlops_orchestrator_stack.py index 59bcd14..3c7d997 100644 --- a/source/lib/mlops_orchestrator_stack.py +++ b/source/lib/mlops_orchestrator_stack.py @@ -100,6 +100,7 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa "accessLogs", encryption=s3.BucketEncryption.S3_MANAGED, block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + versioned=True, ) # Apply secure transfer bucket policy @@ -179,6 +180,7 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa server_access_logs_bucket=access_logs_bucket, server_access_logs_prefix=blueprints_bucket_name, block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + versioned=True, ) # Apply secure transport bucket policy apply_secure_bucket_policy(blueprint_repository_bucket) @@ -329,7 +331,11 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa { "id": "W76", "reason": "A complex IAM policy is required for this resource.", - } + }, + { + "id": "W12", + "reason": "sagemaker:ListModelCards and sagemaker:Search can not have a restricted resource.", + }, ] } } diff --git a/source/requirements-test.txt b/source/requirements-test.txt index 478ae34..2c91643 100644 --- a/source/requirements-test.txt +++ b/source/requirements-test.txt @@ -1,7 +1,6 @@ -sagemaker==2.39.0 -boto3==1.17.23 +sagemaker==2.118.0 +boto3==1.26.20 crhelper==2.0.6 -pytest==6.1.2 -pytest-cov==2.10.1 -moto[all]==2.0.2 -protobuf==3.20.* \ No newline at end of file +pytest==7.2.0 +pytest-cov==4.0.0 +moto[all]==4.0.7 \ No newline at end of file