diff --git a/encrypted-buckets/create-encrypted-buckets-product.yaml b/encrypted-buckets/create-encrypted-buckets-product.yaml index e8122c04..d989c831 100644 --- a/encrypted-buckets/create-encrypted-buckets-product.yaml +++ b/encrypted-buckets/create-encrypted-buckets-product.yaml @@ -369,7 +369,10 @@ Resources: StringEquals: 's3:ExistingObjectTag/servicecatalog:provisioning': 'true' - Effect: Allow - Action: 's3:GetObject' + Action: + - 's3:GetObject' + - 's3:GetObjectTagging' + - 's3:GetObjectVersionTagging' Resource: - 'arn:aws:s3:::sagemaker-*' - Effect: Allow @@ -378,11 +381,13 @@ Resources: - 's3:DeleteBucket' - 's3:DeleteBucketPolicy' - 's3:GetBucketPolicy' + - 's3:GetBucketTagging' - 's3:PutBucketAcl' - 's3:PutBucketNotification' - 's3:PutBucketPolicy' - 's3:PutBucketPublicAccessBlock' - 's3:PutBucketLogging' + - 's3:PutBucketTagging' - 's3:PutEncryptionConfiguration' Resource: 'arn:aws:s3:::sagemaker*' - Action: @@ -503,6 +508,7 @@ Resources: - 'iam:PassRole' Resource: - 'arn:aws:iam::*:role/*custom-project*' + - 'arn:aws:iam::*:role/*SMCustomProject*' Effect: Allow - Effect: Allow Action: @@ -532,7 +538,18 @@ Resources: - Effect: Allow Action: - 's3:*' - Resource: 'arn:aws:s3:::*sagemaker-custom-project-*' + Resource: + - 'arn:aws:s3:::*sagemaker-custom-project-*' + - Effect: Allow + Action: + - 's3:GetObject' + Resource: + - 'arn:aws:s3:::*sagemaker*/*' + - Effect: Allow + Action: + - 's3:ListBucket' + Resource: + - 'arn:aws:s3:::*sagemaker*' - Effect: Allow Action: - 'sagemaker:*' @@ -644,7 +661,7 @@ Resources: PortfolioId: !Ref PortfolioIDParameter ProductId: !Ref ServiceCatalogProduct - ServiceCatalogProductRoleLaunchContstraint: + ServiceCatalogProductRoleLaunchConstraint: Type: AWS::ServiceCatalog::LaunchRoleConstraint DependsOn: - ServiceCatalogProductAssociation diff --git a/encrypted-buckets/project/template.yaml b/encrypted-buckets/project/template.yaml index 9199efa5..01de4e44 100644 --- a/encrypted-buckets/project/template.yaml +++ b/encrypted-buckets/project/template.yaml @@ -93,7 +93,7 @@ Resources: Environment: Type: LINUX_CONTAINER ComputeType: BUILD_GENERAL1_SMALL - Image: 'aws/codebuild/amazonlinux2-x86_64-standard:3.0' + Image: 'aws/codebuild/amazonlinux2-x86_64-standard:5.0' EnvironmentVariables: - Name: SAGEMAKER_PROJECT_NAME Value: !Sub '${SageMakerProjectName}' @@ -231,7 +231,7 @@ Resources: Environment: Type: LINUX_CONTAINER ComputeType: BUILD_GENERAL1_SMALL - Image: 'aws/codebuild/amazonlinux2-x86_64-standard:3.0' + Image: 'aws/codebuild/amazonlinux2-x86_64-standard:5.0' EnvironmentVariables: - Name: SAGEMAKER_PROJECT_NAME Value: !Sub '${SageMakerProjectName}' @@ -266,7 +266,7 @@ Resources: Environment: Type: LINUX_CONTAINER ComputeType: BUILD_GENERAL1_SMALL - Image: 'aws/codebuild/amazonlinux2-x86_64-standard:3.0' + Image: 'aws/codebuild/amazonlinux2-x86_64-standard:5.0' EnvironmentVariables: - Name: SAGEMAKER_PROJECT_NAME Value: !Sub '${SageMakerProjectName}' diff --git a/mlops-multi-account-cdk/mlops-infra/mlops_infra/pipeline_stack.py b/mlops-multi-account-cdk/mlops-infra/mlops_infra/pipeline_stack.py index ad25ac93..404f9418 100644 --- a/mlops-multi-account-cdk/mlops-infra/mlops_infra/pipeline_stack.py +++ b/mlops-multi-account-cdk/mlops-infra/mlops_infra/pipeline_stack.py @@ -111,6 +111,9 @@ def __init__( commands=[], input=pipeline.synth, primary_output_directory="./report", + build_environment=codebuild.BuildEnvironment( + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, + ), partial_build_spec=codebuild.BuildSpec.from_object( { "version": 0.2, @@ -123,7 +126,7 @@ def __init__( }, "phases": { "install": { - "runtime-versions": {"ruby": 3.1}, + "runtime-versions": {"ruby": 3.2, "python": 3.11}, "commands": [ "export date=`date +%Y-%m-%dT%H:%M:%S.%NZ`", "echo Installing cfn_nag - `pwd`", diff --git a/mlops-multi-account-cdk/mlops-infra/scripts/install-prerequisites-brew.sh b/mlops-multi-account-cdk/mlops-infra/scripts/install-prerequisites-brew.sh index a71718d9..2cd05052 100644 --- a/mlops-multi-account-cdk/mlops-infra/scripts/install-prerequisites-brew.sh +++ b/mlops-multi-account-cdk/mlops-infra/scripts/install-prerequisites-brew.sh @@ -4,6 +4,12 @@ # install miniconda to manage python packages brew install --cask miniconda +# conda doesn't initialize from shell, below step to fix that +# https://github.com/conda/conda/issues/7980 +CONDA_BASE=$(conda info --base) +source "$CONDA_BASE"/etc/profile.d/conda.sh +conda init + # install nodejs (required for aws cdk) brew install node diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/pipeline_stack.py b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/pipeline_stack.py index 0f7fcec1..66212cf5 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/pipeline_stack.py +++ b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/pipeline_stack.py @@ -102,6 +102,9 @@ def __init__( commands=[], input=pipeline.synth, primary_output_directory="./report", + build_environment=codebuild.BuildEnvironment( + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, + ), partial_build_spec=codebuild.BuildSpec.from_object( { "version": 0.2, diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/build_pipeline_construct.py b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/build_pipeline_construct.py index a6434c47..4d0dd495 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/build_pipeline_construct.py +++ b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/build_pipeline_construct.py @@ -171,7 +171,7 @@ def __init__( role=codebuild_role, # figure out what actually this role would need build_spec=codebuild.BuildSpec.from_source_filename("buildspec.yml"), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, environment_variables={ "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable(value=project_name), "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable(value=project_id), @@ -215,7 +215,7 @@ def __init__( }, } ), - environment=codebuild.BuildEnvironment(build_image=codebuild.LinuxBuildImage.STANDARD_5_0, privileged=True), + environment=codebuild.BuildEnvironment(build_image=codebuild.LinuxBuildImage.STANDARD_7_0, privileged=True), ) docker_build.add_to_role_policy( diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/deploy_pipeline_construct.py b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/deploy_pipeline_construct.py index 441245ba..04ba4c4a 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/deploy_pipeline_construct.py +++ b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/byoc_pipeline_constructs/deploy_pipeline_construct.py @@ -140,7 +140,7 @@ def __init__( # } # ), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, environment_variables={ "MODEL_PACKAGE_GROUP_NAME": codebuild.BuildEnvironmentVariable(value=model_package_group_name), "PROJECT_ID": codebuild.BuildEnvironmentVariable(value=project_id), @@ -167,7 +167,7 @@ def __init__( }, "phases": { "install": { - "runtime-versions": {"ruby": 2.7}, + "runtime-versions": {"ruby": 3.2}, "commands": [ "export date=`date +%Y-%m-%dT%H:%M:%S.%NZ`", "echo Installing cfn_nag - `pwd`", @@ -192,7 +192,7 @@ def __init__( } ), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, ), ) @@ -291,11 +291,13 @@ def __init__( self, "PreProdActionRole", f"arn:{Aws.PARTITION}:iam::{preprod_account}:role/cdk-hnb659fds-deploy-role-{preprod_account}-{deployment_region}", + mutable=False, ), deployment_role=iam.Role.from_role_arn( self, "PreProdDeploymentRole", f"arn:{Aws.PARTITION}:iam::{preprod_account}:role/cdk-hnb659fds-cfn-exec-role-{preprod_account}-{deployment_region}", + mutable=False, ), cfn_capabilities=[ CfnCapabilities.AUTO_EXPAND, @@ -324,11 +326,13 @@ def __init__( self, "ProdActionRole", f"arn:{Aws.PARTITION}:iam::{prod_account}:role/cdk-hnb659fds-deploy-role-{prod_account}-{deployment_region}", + mutable=False, ), deployment_role=iam.Role.from_role_arn( self, "ProdDeploymentRole", f"arn:{Aws.PARTITION}:iam::{prod_account}:role/cdk-hnb659fds-cfn-exec-role-{prod_account}-{deployment_region}", + mutable=False, ), cfn_capabilities=[ CfnCapabilities.AUTO_EXPAND, diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/build_pipeline_construct.py b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/build_pipeline_construct.py index 9b888af0..82a7ee8f 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/build_pipeline_construct.py +++ b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/build_pipeline_construct.py @@ -178,7 +178,7 @@ def __init__( role=codebuild_role, build_spec=codebuild.BuildSpec.from_source_filename("buildspec.yml"), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, environment_variables={ "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable(value=project_name), "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable(value=project_id), diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/deploy_pipeline_construct.py b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/deploy_pipeline_construct.py index c38ba45d..e3b3dcdd 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/deploy_pipeline_construct.py +++ b/mlops-multi-account-cdk/mlops-sm-project-template/mlops_sm_project_template/templates/pipeline_constructs/deploy_pipeline_construct.py @@ -162,7 +162,7 @@ def __init__( # } # ), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, environment_variables={ "MODEL_PACKAGE_GROUP_NAME": codebuild.BuildEnvironmentVariable(value=model_package_group_name), "PROJECT_ID": codebuild.BuildEnvironmentVariable(value=project_id), @@ -187,7 +187,7 @@ def __init__( }, "phases": { "install": { - "runtime-versions": {"ruby": 2.7}, + "runtime-versions": {"ruby": 3.2}, "commands": [ "export date=`date +%Y-%m-%dT%H:%M:%S.%NZ`", "echo Installing cfn_nag - `pwd`", @@ -212,7 +212,7 @@ def __init__( } ), environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.STANDARD_5_0, + build_image=codebuild.LinuxBuildImage.STANDARD_7_0, ), ) @@ -311,11 +311,13 @@ def __init__( self, "PreProdActionRole", f"arn:{Aws.PARTITION}:iam::{preprod_account}:role/cdk-hnb659fds-deploy-role-{preprod_account}-{deployment_region}", + mutable=False, ), deployment_role=iam.Role.from_role_arn( self, "PreProdDeploymentRole", f"arn:{Aws.PARTITION}:iam::{preprod_account}:role/cdk-hnb659fds-cfn-exec-role-{preprod_account}-{deployment_region}", + mutable=False, ), cfn_capabilities=[ CfnCapabilities.AUTO_EXPAND, @@ -344,11 +346,13 @@ def __init__( self, "ProdActionRole", f"arn:{Aws.PARTITION}:iam::{prod_account}:role/cdk-hnb659fds-deploy-role-{prod_account}-{deployment_region}", + mutable=False, ), deployment_role=iam.Role.from_role_arn( self, "ProdDeploymentRole", f"arn:{Aws.PARTITION}:iam::{prod_account}:role/cdk-hnb659fds-cfn-exec-role-{prod_account}-{deployment_region}", + mutable=False, ), cfn_capabilities=[ CfnCapabilities.AUTO_EXPAND, diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/scripts/install-prerequisites-brew.sh b/mlops-multi-account-cdk/mlops-sm-project-template/scripts/install-prerequisites-brew.sh index a71718d9..2cd05052 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/scripts/install-prerequisites-brew.sh +++ b/mlops-multi-account-cdk/mlops-sm-project-template/scripts/install-prerequisites-brew.sh @@ -4,6 +4,12 @@ # install miniconda to manage python packages brew install --cask miniconda +# conda doesn't initialize from shell, below step to fix that +# https://github.com/conda/conda/issues/7980 +CONDA_BASE=$(conda info --base) +source "$CONDA_BASE"/etc/profile.d/conda.sh +conda init + # install nodejs (required for aws cdk) brew install node diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_build_app/buildspec.yml b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_build_app/buildspec.yml index 29fc658f..4b4746dd 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_build_app/buildspec.yml +++ b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_build_app/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.8 + python: 3.11 commands: - pip install --upgrade --force-reinstall . "awscli>1.20.30" diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_deploy_app/buildspec.yml b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_deploy_app/buildspec.yml index 9127d73b..1b9c2b04 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_deploy_app/buildspec.yml +++ b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/batch_deploy_app/buildspec.yml @@ -1,6 +1,9 @@ version: 0.2 phases: + install: + runtime-versions: + python: 3.11 build: commands: - npm install -g aws-cdk diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/build_app/buildspec.yml b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/build_app/buildspec.yml index 9f9010d1..996daf6b 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/build_app/buildspec.yml +++ b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/build_app/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.8 + python: 3.11 commands: - pip install --upgrade --force-reinstall . "awscli>1.20.30" diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/byoc_build_app/buildspec.yml b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/byoc_build_app/buildspec.yml index b94ff07c..fde81c19 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/byoc_build_app/buildspec.yml +++ b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/byoc_build_app/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.8 + python: 3.11 commands: - pip install --upgrade --force-reinstall . "awscli>1.20.30" diff --git a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/deploy_app/buildspec.yml b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/deploy_app/buildspec.yml index 9127d73b..1b9c2b04 100644 --- a/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/deploy_app/buildspec.yml +++ b/mlops-multi-account-cdk/mlops-sm-project-template/seed_code/deploy_app/buildspec.yml @@ -1,6 +1,9 @@ version: 0.2 phases: + install: + runtime-versions: + python: 3.11 build: commands: - npm install -g aws-cdk