diff --git a/.github/workflows/build-deploy-pipeline.yml b/.github/workflows/build-deploy-pipeline.yml new file mode 100644 index 0000000..0ff1539 --- /dev/null +++ b/.github/workflows/build-deploy-pipeline.yml @@ -0,0 +1,373 @@ +# Create a unified Build and Deploy pipeline that uses approval gates +# Build Model –> Deploy to staging with approval –> Deploy to prod with approval. +# https://timheuer.com/blog/add-approval-workflow-to-github-actions/ + +name: Build and Deploy + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + pull_request: + branches: + - main + release: + types: + - created + +env: + AWS_REGION: us-east-1 + PROJECT_NAME: ${{ github.event.repository.name }} + +jobs: + build: + name: Build Model + runs-on: ubuntu-latest + environment: + name: development + defaults: + run: + shell: bash + working-directory: ./build_pipeline + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: "12" + cache: npm + + - name: Install Requirements + run: | + npm install -g aws-cdk # Install cdk + pip install --requirement requirements.txt + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 1200 + + - name: Build Pipeline + id: build-pipeline + env: + SAGEMAKER_PROJECT_NAME: ${{ env.PROJECT_NAME }} + SAGEMAKER_PIPELINE_NAME: ${{ env.PROJECT_NAME }}-pipeline + SAGEMAKER_PIPELINE_DESCRIPTION: "Drift detection model build pipeline created from GitHub actions" + SAGEMAKER_PIPELINE_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + run: | + export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text` + echo "Project id: $SAGEMAKER_PROJECT_ID" + export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION + echo "Artifact Bucket: $ARTIFACT_BUCKET" + npx cdk synth --path-metadata false --asset-metadata=false > drift-pipeline.yml + echo "::set-output name=pipeline_name::$SAGEMAKER_PIPELINE_NAME" + + - name: Print template + run: cat drift-pipeline.yml + + - name: Create CFN Pipeline + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: sagemaker-${{ env.PROJECT_NAME }}-pipeline + template: ./build_pipeline/drift-pipeline.yml # Need to specify working-directory + role-arn: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + no-fail-on-empty-changeset: "1" + + - name: Start Pipeline + run: aws sagemaker start-pipeline-execution --pipeline-name ${{ steps.build-pipeline.outputs.pipeline_name }} --pipeline-parameters Name=InputSource,Value=GitHubAction#${{ github.run_number }} + + - name: Upload template + uses: actions/upload-artifact@v2 + with: + name: drift-pipeline + path: ./build_pipeline/drift-pipeline.yml + + batch_staging: + needs: build + name: Batch to staging + runs-on: ubuntu-latest + environment: + name: batch-staging + defaults: + run: + shell: bash + working-directory: ./batch_pipeline + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: "12" + cache: npm + + - name: Install Requirements + run: | + npm install -g aws-cdk # Install cdk + pip install --requirement requirements.txt + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + id: creds + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 1200 + + - name: Build Templates + id: build-templates + env: + SAGEMAKER_PROJECT_NAME: ${{ env.PROJECT_NAME }} + SAGEMAKER_PIPELINE_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + LAMBDA_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + run: | + export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text` + echo "Project id: $SAGEMAKER_PROJECT_ID" + export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION + echo "Artifact Bucket: $ARTIFACT_BUCKET" + npx cdk synth drift-batch-staging --path-metadata false --asset-metadata=false > drift-batch-staging.yml + + - name: Print template + run: cat drift-batch-staging.yml + + - name: Deploy Staging + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: sagemaker-${{ env.PROJECT_NAME }}-batch-staging + template: ./batch_pipeline/drift-batch-staging.yml # Need to specify working-directory + role-arn: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + no-fail-on-empty-changeset: "1" + + - name: Upload template + uses: actions/upload-artifact@v2 + with: + name: drift-batch-staging + path: ./batch_pipeline/drift-batch-staging.yml + + batch_prod: + needs: batch_staging + name: Batch to prod + if: ${{ github.ref == 'refs/heads/main' }} # Filter to only run on main branch + runs-on: ubuntu-latest + environment: + name: batch-prod + defaults: + run: + shell: bash + working-directory: ./batch_pipeline + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: "12" + cache: npm + + - name: Install Requirements + run: | + npm install -g aws-cdk # Install cdk + pip install --requirement requirements.txt + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 1200 + + - name: Build Templates + id: build-templates + env: + SAGEMAKER_PROJECT_NAME: ${{ env.PROJECT_NAME }} + SAGEMAKER_PIPELINE_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + LAMBDA_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + run: | + export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text` + echo "Project id: $SAGEMAKER_PROJECT_ID" + export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION + echo "Artifact Bucket: $ARTIFACT_BUCKET" + npx cdk synth drift-batch-prod --path-metadata false --asset-metadata=false > drift-batch-prod.yml + + - name: Print Template + run: cat drift-batch-prod.yml + + - name: Deploy Prod + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: sagemaker-${{ env.PROJECT_NAME }}-batch-prod + template: ./batch_pipeline/drift-batch-prod.yml # Need to specify working-directory + role-arn: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + no-fail-on-empty-changeset: "1" + + - name: Upload template + uses: actions/upload-artifact@v2 + with: + name: drift-batch-prod + path: ./batch_pipeline/drift-batch-prod.yml + + deploy_staging: + needs: build + name: Deploy to staging + runs-on: ubuntu-latest + environment: + name: staging # Use different environment that optionally requires approval + defaults: + run: + shell: bash + working-directory: ./deployment_pipeline + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: "12" + cache: npm + + - name: Install Requirements + run: | + npm install -g aws-cdk # Install cdk + pip install --requirement requirements.txt + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 1200 + + - name: Build Templates + id: build-templates + env: + SAGEMAKER_PROJECT_NAME: ${{ env.PROJECT_NAME }} + SAGEMAKER_EXECUTION_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + run: | + export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text` + echo "Project id: $SAGEMAKER_PROJECT_ID" + export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION + echo "Artifact Bucket: $ARTIFACT_BUCKET" + npx cdk synth drift-deploy-staging --path-metadata false --asset-metadata=false > drift-deploy-staging.yml + + - name: Print template + run: cat drift-deploy-staging.yml + + - name: Deploy Staging + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: sagemaker-${{ env.PROJECT_NAME }}-deploy-staging + template: ./deployment_pipeline/drift-deploy-staging.yml # Need to specify working-directory + role-arn: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + no-fail-on-empty-changeset: "1" + + - name: Upload template + uses: actions/upload-artifact@v2 + with: + name: drift-deploy-staging + path: ./deployment_pipeline/drift-deploy-staging.yml + + deploy_prod: + needs: deploy_staging + name: Deploy to prod + if: ${{ github.ref == 'refs/heads/main' }} # Filter to only run on main branch + runs-on: ubuntu-latest + environment: + name: prod # Use different environment that requires approval + defaults: + run: + shell: bash + working-directory: ./deployment_pipeline + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: "12" + cache: npm + + - name: Install Requirements + run: | + npm install -g aws-cdk # Install cdk + pip install --requirement requirements.txt + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 1200 + + - name: Build Templates + id: build-templates + env: + SAGEMAKER_PROJECT_NAME: ${{ env.PROJECT_NAME }} + SAGEMAKER_EXECUTION_ROLE_ARN: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + run: | + export SAGEMAKER_PROJECT_ID=`aws sagemaker describe-project --project-name $SAGEMAKER_PROJECT_NAME --query ProjectId --output text` + echo "Project id: $SAGEMAKER_PROJECT_ID" + export ARTIFACT_BUCKET=sagemaker-project-$SAGEMAKER_PROJECT_ID-$AWS_REGION + echo "Artifact Bucket: $ARTIFACT_BUCKET" + npx cdk synth drift-deploy-prod --path-metadata false --asset-metadata=false > drift-deploy-prod.yml + + - name: Print Template + run: cat drift-deploy-prod.yml + + - name: Deploy Prod + id: deploy-pipeline + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: sagemaker-${{ env.PROJECT_NAME }}-deploy-prod + template: ./deployment_pipeline/drift-deploy-prod.yml # Need to specify working-directory + role-arn: arn:aws:iam::${{ steps.creds.outputs.aws-account-id }}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + no-fail-on-empty-changeset: "1" + + - name: Upload template + uses: actions/upload-artifact@v2 + with: + name: drift-deploy-prod + path: ./deployment_pipeline/drift-deploy-prod.yml \ No newline at end of file diff --git a/.github/workflows/publish-template.yml b/.github/workflows/publish-template.yml index e418616..e659e95 100644 --- a/.github/workflows/publish-template.yml +++ b/.github/workflows/publish-template.yml @@ -3,12 +3,12 @@ name: Publish Template to S3 on: push: branches: [ main ] - pull_request: - branches: [ main ] jobs: synth-and-publish: runs-on: ubuntu-latest + environment: + name: aws # Target only the AWS environment to publish CFN templates defaults: run: shell: bash @@ -19,8 +19,8 @@ jobs: - name: Setup Python environment uses: actions/setup-python@v2 with: - python-version: "3.8" # Version range or exact version of a Python version to use, using SemVer's version range syntax - architecture: "x64" # optional x64 or x86. Defaults to x64 if not specified + python-version: "3.8" + architecture: "x64" - name: Setup Node uses: actions/setup-node@v2 @@ -47,7 +47,7 @@ jobs: env: BUCKET_NAME: ${{ secrets.BUCKET_NAME }} BUCKET_PREFIX: ${{ secrets.BUCKET_PREFIX }} - run: cdk synth drift-service-catalog --path-metadata false -c drift:ArtifactBucket=$BUCKET_NAME -c drift:ArtifactBucketPrefix=$BUCKET_PREFIX > drift-service-catalog.yml + run: cdk synth drift-service-catalog --path-metadata false -c drift:ArtifactBucket=$BUCKET_NAME -c drift:ArtifactBucketPrefix=$BUCKET_PREFIX > cloudformation/drift-service-catalog.yml - name: Publish Assets to S3 env: @@ -62,6 +62,6 @@ jobs: path: cdk.out/*.template.json - name: Print Template - run: cat drift-service-catalog.yml + run: cat cloudformation/drift-service-catalog.yml diff --git a/ACTIONS.md b/ACTIONS.md new file mode 100644 index 0000000..ccfb690 --- /dev/null +++ b/ACTIONS.md @@ -0,0 +1,59 @@ +# AWS SageMaker Workflow for GitHub Actions + +This template repository contains a sample application and sample GitHub Actions workflow files for continuously deploying both application code and infrastructure as code with GitHub Actions. + +This MLOps workflow demonstrates training and evaluating a machine learning model to predict taxi fare from the public [New York City Taxi dataset](https://registry.opendata.aws/nyc-tlc-trip-records-pds/) deployed with Amazon SageMaker. + +This repository contains a number of start workflow files for GitHub Actions: +1. [build-deploy-pipeline.yml](.github/workflows/build-deploy-pipeline.yml) This workflow runs when a pull request is opened or pushed to the `main` branch (see below). +1. [publish-template.yml](.github/workflows/publish-template.yml) runs when a new commit is pushed to the main branch in the `aws` environment. + +## Create a GitHub repository from this template + +Click the "Use this template" button above to create a new repository from this template. + +Clone your new repository, and deploy the IAM resources needed to enable GitHub Actions to deploy CloudFormation templates: + +``` +aws cloudformation deploy \ + --stack-name amazon-sagemaker-workflow-for-github-actions \ + --template-file cloudformation/github-actions-setup.yml \ + --capabilities CAPABILITY_NAMED_IAM \ + --region us-east-1 +``` +You can review the permissions that your repository's GitHub Actions deployment workflow will have in the [github-actions-setup.yml](cloudformation/github-actions-setup.yml) CloudFormation template. + +Retrieve the IAM access key credentials that GitHub Actions will use for deployments: +``` +aws secretsmanager get-secret-value \ + --secret-id github-actions-sagemaker \ + --region us-east-1 \ + --query SecretString \ + --output text +``` + +### Build Deploy Pipeline + +This sample includes a `Build and Deploy` [GitHub Actions](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions) workflow with contains the following jobs +1. The `Build Model` job will create or update a SageMaker Model Build Pipeline using AWS CloudFormation. +2. The `Batch` jobs will create or update a SageMaker Pipeline to run Batch Scoring for Staging and Production environments. +3. The `Deploy` jobs will deploy SageMaker Endpoints to Staging and Production environments. + +These jobs are configured to run against a specific [environment](https://docs.github.com/en/actions/reference/environments) which contains both secrets and optional *protection rules*. + +![Execution Role](docs/github-actions-workflow.png) + +### Environments + +1. `development` environment in which runs your `Build Model` job and starts the SageMaker pipeline execution. On completion this pipeline will publish a model to the Registry. It is recommend you run this on `pull_request` and `push` events. +2. `staging` and `batch-staging` environments will enable you to run the `Batch` and `Deploy` jobs respectively in staging. + * You should configure a *protection rule* for data science team so this job is delayed until the latest model has been approved in the SageMaker model registry. + 3. `prod` and `batch-prod` environments will enable you to run the `Batch` and `Deploy` jobs respectively in production. + * You should configure a *protection rule* for your operations team which will approve this only once they are happy that the staging environment has been tested. + +For each of the environments you will require setting up the following secrets. +1. Create a secret named `AWS_ACCESS_KEY_ID` containing the `AccessKeyId` value returned above. +1. Create a secret named `AWS_SECRET_ACCESS_KEY` containing in the `SecretAccessKey` value returned above. +1. Create a secret named `AWS_SAGEMAKER_ROLE` containing the ARN for the `AmazonSageMakerServiceCatalogProductsUseRole` in your account. + +When the workflow successfully completes, drift detection is configured to trigger re-training on drift detection in the production batch pipeline or real-time endpoint. diff --git a/README.md b/README.md index 2f48d52..fba9a6d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Once complete, you can Train and Deploy machine learning models, and send traffi Use this following AWS CloudFormation quick start to create a custom [SageMaker MLOps project](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-projects-templates-custom.html) template in the [AWS Service Catalog](https://aws.amazon.com/servicecatalog/) and configure the portfolio and products so you can launch the project from within your Studio domain. -[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateUrl=https%3A%2F%2Famazon-sagemaker-safe-deployment-pipeline.s3.amazonaws.com%2Fdrift-pipeline%2Fdrift-service-catalog.yml&stackName=drift-pipeline¶m_ExecutionRoleArn=¶m_PortfolioName=SageMaker%20Organization%20Templates¶m_PortfolioOwner=administrator¶m_ProductVersion=1.0) +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateUrl=https%3A%2F%2Famazon-sagemaker-safe-deployment-pipeline.s3.amazonaws.com%2Fdrift-pipeline%2Fcloudformation%2Fdrift-service-catalog.yml&stackName=drift-pipeline¶m_ExecutionRoleArn=¶m_PortfolioName=SageMaker%20Organization%20Templates¶m_PortfolioOwner=administrator¶m_ProductVersion=1.0) Follow are the list of the parameters. diff --git a/batch_pipeline/app.py b/batch_pipeline/app.py index 51cf5b1..2665ee1 100644 --- a/batch_pipeline/app.py +++ b/batch_pipeline/app.py @@ -27,8 +27,8 @@ def create_pipeline( project_id: str, region: str, sagemaker_pipeline_role_arn: str, + lambda_role_arn: str, artifact_bucket: str, - evaluate_drift_function_arn: str, stage_name: str, ): # Get the stage specific deployment config for sagemaker @@ -85,7 +85,7 @@ def create_pipeline( pipeline_name=sagemaker_pipeline_name, default_bucket=artifact_bucket, base_job_prefix=project_id, - evaluate_drift_function_arn=evaluate_drift_function_arn, + lambda_role_arn=lambda_role_arn, data_uri=data_uri, model_uri=model_uri, transform_uri=transform_uri, @@ -98,8 +98,7 @@ def create_pipeline( parsed = json.loads(pipeline_definition_body) logger.info(json.dumps(parsed, indent=2, sort_keys=True)) - # Upload the pipeline to S3 bucket/key and return JSON with key/value for for Cfn Stack parameters. - # see: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sagemaker-pipeline.html + # Upload the pipeline to S3 bucket/key logger.info(f"Uploading {stage_name} pipeline to {artifact_bucket}") pipeline_definition_key = upload_pipeline( pipeline, @@ -131,8 +130,8 @@ def main( project_id: str, region: str, sagemaker_pipeline_role_arn: str, + lambda_role_arn: str, artifact_bucket: str, - evaluate_drift_function_arn: str, ): # Create App and stacks app = core.App() @@ -143,8 +142,8 @@ def main( project_id=project_id, region=region, sagemaker_pipeline_role_arn=sagemaker_pipeline_role_arn, + lambda_role_arn=lambda_role_arn, artifact_bucket=artifact_bucket, - evaluate_drift_function_arn=evaluate_drift_function_arn, stage_name="staging", ) @@ -154,8 +153,8 @@ def main( project_id=project_id, region=region, sagemaker_pipeline_role_arn=sagemaker_pipeline_role_arn, + lambda_role_arn=lambda_role_arn, artifact_bucket=artifact_bucket, - evaluate_drift_function_arn=evaluate_drift_function_arn, stage_name="prod", ) @@ -175,8 +174,8 @@ def main( default=os.environ.get("SAGEMAKER_PIPELINE_ROLE_ARN"), ) parser.add_argument( - "--evaluate-drift-function-arn", - default=os.environ.get("EVALUATE_DRIFT_FUNCTION_ARN"), + "--lambda-role-arn", + default=os.environ.get("LAMBDA_ROLE_ARN"), ) parser.add_argument( "--artifact-bucket", diff --git a/batch_pipeline/infra/sagemaker_pipeline_stack.py b/batch_pipeline/infra/sagemaker_pipeline_stack.py index dfcd146..51cc019 100644 --- a/batch_pipeline/infra/sagemaker_pipeline_stack.py +++ b/batch_pipeline/infra/sagemaker_pipeline_stack.py @@ -1,14 +1,12 @@ from aws_cdk import ( core, aws_cloudwatch as cloudwatch, - aws_events as events, aws_iam as iam, + aws_lambda as lambda_, aws_sagemaker as sagemaker, ) import logging -import os -from urllib.parse import urlparse from batch_config import DriftConfig logger = logging.getLogger(__name__) diff --git a/lambda/batch/lambda_evaluate_drift.py b/batch_pipeline/lambda/lambda_evaluate_drift.py similarity index 100% rename from lambda/batch/lambda_evaluate_drift.py rename to batch_pipeline/lambda/lambda_evaluate_drift.py diff --git a/batch_pipeline/pipelines/pipeline.py b/batch_pipeline/pipelines/pipeline.py index ada00d2..c5236d6 100644 --- a/batch_pipeline/pipelines/pipeline.py +++ b/batch_pipeline/pipelines/pipeline.py @@ -73,7 +73,7 @@ def get_pipeline( pipeline_name: str, default_bucket: str, base_job_prefix: str, - evaluate_drift_function_arn: str, + lambda_role_arn: str, data_uri: str, model_uri: str, transform_uri: str, @@ -241,10 +241,15 @@ def get_pipeline( cache_config=cache_config, ) - # Create a lambda step that inspects the output of the model monitoring + # Create an inline lambda step that inspects the output of the model monitoring step_lambda = LambdaStep( name="EvaluateDrift", - lambda_func=Lambda(function_arn=evaluate_drift_function_arn), + lambda_func=Lambda( + function_name=f"sagemaker-{pipeline_name}", # Must be <64 characters + execution_role_arn=lambda_role_arn, + script=os.path.join(BASE_DIR, "../lambda/lambda_evaluate_drift.py"), + handler="lambda_evaluate_drift.lambda_handler", + ), inputs={ "ProcessingJobName": step_monitor.properties.ProcessingJobName, "PipelineName": pipeline_name, @@ -256,8 +261,6 @@ def get_pipeline( ], ) - # TODO: Fail workflow when statusCode==400 - steps += [step_monitor, step_lambda] # pipeline instance diff --git a/batch_pipeline/setup.py b/batch_pipeline/setup.py index 21a8342..81d740e 100644 --- a/batch_pipeline/setup.py +++ b/batch_pipeline/setup.py @@ -20,6 +20,7 @@ "aws-cdk.aws-cloudwatch==1.116.0", "aws-cdk.aws-events==1.116.0", "aws-cdk.aws-iam==1.116.0", + "aws-cdk.aws-lambda==1.116.0", "aws-cdk.aws-sagemaker==1.116.0", "sagemaker==2.54.0", ], diff --git a/build_pipeline/app.py b/build_pipeline/app.py index 68896a9..b1f725a 100644 --- a/build_pipeline/app.py +++ b/build_pipeline/app.py @@ -23,7 +23,6 @@ def main( sagemaker_pipeline_description, sagemaker_pipeline_role, artifact_bucket, - output_dir, ): # Use project_name for pipeline and model package group name model_package_group_name = project_name @@ -36,29 +35,20 @@ def main( base_job_prefix=project_id, ) - # Create output directory - if not os.path.exists(output_dir): - os.mkdir(output_dir) - # Create the pipeline definition logger.info("Creating/updating a SageMaker Pipeline") pipeline_definition_body = pipeline.definition() parsed = json.loads(pipeline_definition_body) logger.debug(json.dumps(parsed, indent=2, sort_keys=True)) - # Upload the pipeline to S3 bucket/key and return JSON with key/value for for Cfn Stack parameters. + # Upload the pipeline to S3 bucket and return the target key # see: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sagemaker-pipeline.html - pipeline_location = upload_pipeline( + pipeline_definition_key = upload_pipeline( pipeline, default_bucket=artifact_bucket, base_job_prefix=f"{project_id}/build", ) - # Store parameters as template-config.json used in the next CodePipeline step to create the SageMakerPipelineStack. - with open(os.path.join(output_dir, "template-config.json"), "w") as f: - template_configuration = {"Parameters": pipeline_location} - json.dump(template_configuration, f) - # Create App and stacks app = core.App() @@ -73,7 +63,8 @@ def main( model_package_group_name=model_package_group_name, pipeline_name=sagemaker_pipeline_name, pipeline_description=sagemaker_pipeline_description, - # pipeline_definition_body=pipeline_definition_body, + pipeline_definition_bucket=artifact_bucket, + pipeline_definition_key=pipeline_definition_key, role_arn=sagemaker_pipeline_role, tags=tags, ) @@ -105,7 +96,6 @@ def main( "--artifact-bucket", default=os.environ.get("ARTIFACT_BUCKET"), ) - parser.add_argument("--output-dir", default="dist") args = vars(parser.parse_args()) - print("args: {}".format(args)) + logger.info("args: {}".format(args)) main(**args) diff --git a/build_pipeline/infra/sagemaker_pipeline_stack.py b/build_pipeline/infra/sagemaker_pipeline_stack.py index 830c56b..d626b11 100644 --- a/build_pipeline/infra/sagemaker_pipeline_stack.py +++ b/build_pipeline/infra/sagemaker_pipeline_stack.py @@ -19,28 +19,14 @@ def __init__( model_package_group_name: str, pipeline_name: str, pipeline_description: str, - # pipeline_definition_body: str, + pipeline_definition_bucket: str, + pipeline_definition_key: str, role_arn: str, tags: list, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) - definition_bucket = core.CfnParameter( - self, - "PipelineDefinitionBucket", - type="String", - description="The s3 bucket for pipeline definition", - min_length=1, - ) - definition_key = core.CfnParameter( - self, - "PipelineDefinitionKey", - type="String", - description="The s3 key for pipeline definition", - min_length=1, - ) - sagemaker.CfnModelPackageGroup( self, "ModelPackageGroup", @@ -56,11 +42,10 @@ def __init__( pipeline_description=pipeline_description, pipeline_definition={ "PipelineDefinitionS3Location": { - "Bucket": definition_bucket.value_as_string, - "Key": definition_key.value_as_string, + "Bucket": pipeline_definition_bucket, + "Key": pipeline_definition_key, } }, - # pipeline_definition={"PipelineDefinitionBody": pipeline_definition_body}, role_arn=role_arn, tags=tags, ) diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index 50e9b77..90b9e3b 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -81,7 +81,7 @@ def get_pipeline( model_package_group_name, default_bucket, base_job_prefix, -): +) -> Pipeline: """Gets a SageMaker ML Pipeline instance working with on nyc taxi data. Args: region: AWS region to create and run the pipeline. @@ -381,16 +381,12 @@ def get_pipeline( return pipeline -def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix): +def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix) -> str: # Get the pipeline definition pipeline_definition_body = pipeline.definition() # Upload the pipeline to a unique location in s3 based on git commit and timestamp - pipeline_name = name_from_base(f"{base_job_prefix}/pipeline") + pipeline_key = name_from_base(f"{base_job_prefix}/pipeline.json") S3Uploader.upload_string_as_file_body( - pipeline_definition_body, f"s3://{default_bucket}/{pipeline_name}.json" + pipeline_definition_body, f"s3://{default_bucket}/{pipeline_key}" ) - # Return JSON with parameters used in Cfn Stack creation as template-configuration.json - return { - "PipelineDefinitionBucket": default_bucket, - "PipelineDefinitionKey": f"{pipeline_name}.json", - } + return pipeline_key diff --git a/drift-service-catalog.yml b/cloudformation/drift-service-catalog.yml similarity index 99% rename from drift-service-catalog.yml rename to cloudformation/drift-service-catalog.yml index 2c06cd4..058de25 100644 --- a/drift-service-catalog.yml +++ b/cloudformation/drift-service-catalog.yml @@ -401,4 +401,3 @@ Conditions: - Fn::Equals: - Ref: AWS::Region - us-west-2 - diff --git a/cloudformation/github-actions-setup.yml b/cloudformation/github-actions-setup.yml new file mode 100644 index 0000000..90480b5 --- /dev/null +++ b/cloudformation/github-actions-setup.yml @@ -0,0 +1,102 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Setup IAM user for GitHub Actions and IAM role for SageMaker + +Parameters: + GitHubActionsUserName: + Type: String + Default: github-actions-sagemaker + +Resources: + GitHubActionsUser: + Type: AWS::IAM::User + Properties: + UserName: !Ref GitHubActionsUserName + + GitHubActionsUserAccessKey: + Type: AWS::IAM::AccessKey + Properties: + UserName: !Ref GitHubActionsUser + Serial: 1 + + GitHubActionsCredentials: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Ref GitHubActionsUserName + SecretString: !Sub | + { + "AccessKeyId":"${GitHubActionsUserAccessKey}", + "SecretAccessKey":"${GitHubActionsUserAccessKey.SecretAccessKey}" + } + + # Permissions to put sagemaker resources to s3 bucket and run CloudFormation passing SageMaker role + GitHubActionsDeployPolicy: + Type: AWS::IAM::Policy + Properties: + Users: + - !Ref GitHubActionsUser + PolicyName: allow-github-actions + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: + - "cloudformation:*" + Effect: Allow + Resource: "*" + - Action: "cloudformation:DeleteStack" + Effect: Deny + Resource: "*" + - Action: + - "s3:CreateBucket" + - "s3:GetBucket*" + - "s3:ListAllMyBuckets" + - "s3:ListBucket" + - "s3:GetObject*" + - "s3:PutObject*" + Effect: Allow + Resource: "arn:aws:s3:::sagemaker-*" + - Action: + - "sagemaker:Describe*" + - "sagemaker:List*" + - "sagemaker:Search" + - "sagemaker:StartPipelineExecution" + Effect: Allow + Resource: "*" + - Action: "iam:PassRole" + Effect: Allow + Resource: !Sub arn:aws:iam::${AWS::AccountId}:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole + - Action: + - "lambda:*" + Effect: Allow + Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:sagemaker-*" + + LambdaPolicy: + Type: AWS::IAM::Policy + Properties: + Roles: + - AmazonSageMakerServiceCatalogProductsUseRole + PolicyName: allow-cloudwatch-alarms + PolicyDocument: + Version: "2012-10-17" + Statement: + - Action: + - "cloudwatch:*" + Effect: Allow + Resource: !Sub "arn:aws:cloudwatch:${AWS::Region}:${AWS::AccountId}:alarm:sagemaker-*" + - Action: + - "cloudwatch:*" + Effect: Allow + Resource: !Sub "arn:aws:cloudwatch:${AWS::Region}:${AWS::AccountId}:alarm:sagemaker-*" + - Action: + - "application-autoscaling:DeregisterScalableTarget" + - "application-autoscaling:DeleteScalingPolicy" + - "application-autoscaling:DescribeScalingPolicies" + - "application-autoscaling:PutScalingPolicy" + - "application-autoscaling:DescribeScalingPolicies" + - "application-autoscaling:RegisterScalableTarget" + - "application-autoscaling:DescribeScalableTargets" + - "iam:CreateServiceLinkedRole" + - "cloudwatch:DeleteAlarms" + - "cloudwatch:DescribeAlarms" + - "cloudwatch:PutMetricAlarm" + Effect: Allow + Resource: "*" diff --git a/deployment_pipeline/app.py b/deployment_pipeline/app.py index 8e5f19f..7f4336c 100644 --- a/deployment_pipeline/app.py +++ b/deployment_pipeline/app.py @@ -143,5 +143,5 @@ def main( default=os.environ.get("ARTIFACT_BUCKET"), ) args = vars(parser.parse_args()) - print("args: {}".format(args)) + logger.info("args: {}".format(args)) main(**args) diff --git a/docs/github-actions-workflow.png b/docs/github-actions-workflow.png new file mode 100644 index 0000000..36d9206 Binary files /dev/null and b/docs/github-actions-workflow.png differ diff --git a/infra/batch_pipeline_construct.py b/infra/batch_pipeline_construct.py index 5fbbbf0..b67fdfd 100644 --- a/infra/batch_pipeline_construct.py +++ b/infra/batch_pipeline_construct.py @@ -81,25 +81,6 @@ def __init__( ) ) - # Load the start pipeline code - with open("lambda/batch/lambda_evaluate_drift.py", encoding="utf8") as fp: - lambda_evaluate_drift_code = fp.read() - - lambda_evaluate_drift = lambda_.Function( - self, - "EvaluateDriftFunction", - function_name=f"sagemaker-{project_name}-evaluate-drift", - code=lambda_.Code.from_inline(lambda_evaluate_drift_code), - role=lambda_role, - handler="index.lambda_handler", - runtime=lambda_.Runtime.PYTHON_3_8, - timeout=core.Duration.seconds(3), - memory_size=128, - environment={ - "LOG_LEVEL": "INFO", - }, - ) - # Define AWS CodeBuild spec to run node.js and python # https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html pipeline_build = codebuild.PipelineProject( @@ -156,12 +137,12 @@ def __init__( "SAGEMAKER_PIPELINE_ROLE_ARN": codebuild.BuildEnvironmentVariable( value=sagemaker_execution_role.role_arn, ), + "LAMBDA_ROLE_ARN": codebuild.BuildEnvironmentVariable( + value=lambda_role.role_arn, + ), "ARTIFACT_BUCKET": codebuild.BuildEnvironmentVariable( value=s3_artifact.bucket_name ), - "EVALUATE_DRIFT_FUNCTION_ARN": codebuild.BuildEnvironmentVariable( - value=lambda_evaluate_drift.function_arn, - ), }, ), ) @@ -261,7 +242,7 @@ def __init__( ) # Load the lambda pipeline change code - with open("lambda/build/lambda_pipeline_change.py", encoding="utf8") as fp: + with open("lambda/lambda_pipeline_change.py", encoding="utf8") as fp: lambda_pipeline_change_code = fp.read() lambda_pipeline_change = lambda_.Function( diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index 194570a..f4c354f 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -114,7 +114,6 @@ def __init__( "base-directory": "dist", "files": [ "pipeline.json", - "template-config.json", "*.template.json", ], }, @@ -147,7 +146,7 @@ def __init__( ) # Load the start pipeline code - with open("lambda/build/lambda_start_pipeline.py", encoding="utf8") as fp: + with open("lambda/lambda_start_pipeline.py", encoding="utf8") as fp: lambda_start_pipeline_code = fp.read() lambda_start_pipeline = lambda_.Function( @@ -223,9 +222,6 @@ def __init__( template_path=pipeline_build_output.at_path( "drift-sagemaker-pipeline.template.json" ), - template_configuration=pipeline_build_output.at_path( - "template-config.json" - ), stack_name="sagemaker-{}-pipeline".format(project_name), admin_permissions=False, deployment_role=cloudformation_role, @@ -292,7 +288,7 @@ def __init__( ) # Load the lambda pipeline change code - with open("lambda/build/lambda_pipeline_change.py", encoding="utf8") as fp: + with open("lambda/lambda_pipeline_change.py", encoding="utf8") as fp: lambda_pipeline_change_code = fp.read() lambda_pipeline_change = lambda_.Function( diff --git a/infra/pipeline_stack.py b/infra/pipeline_stack.py index c4b5f96..c553da1 100644 --- a/infra/pipeline_stack.py +++ b/infra/pipeline_stack.py @@ -297,7 +297,7 @@ def resolve_ssm_parameter(self, key: str): class BatchPipelineStack(PipelineStack): - """Creates a Pipelinf for real-time deployment""" + """Creates a Pipeline for batch deployment""" def __init__( self, @@ -309,7 +309,7 @@ def __init__( class DeployPipelineStack(PipelineStack): - """Creates a Pipelinf for real-time deployment""" + """Creates a Pipelinfe for real-time deployment""" def __init__( self, diff --git a/infra/service_catalog_stack.py b/infra/service_catalog_stack.py index d751336..99db61e 100644 --- a/infra/service_catalog_stack.py +++ b/infra/service_catalog_stack.py @@ -149,6 +149,18 @@ def __init__( ) ) + # Add permissions to get/create lambda in batch pipeline + products_use_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "lambda:*", + ], + resources=[ + f"arn:aws:lambda:{self.region}:{self.account}:function:sagemaker-*" + ], + ) + ) + portfolio = servicecatalog.Portfolio( self, "Portfolio", diff --git a/infra/upload_assets.py b/infra/upload_assets.py index bb0b8e5..da12369 100644 --- a/infra/upload_assets.py +++ b/infra/upload_assets.py @@ -121,7 +121,7 @@ def upload_assets(cdk_dir: str = "cdk.out") -> None: logger.addHandler(ch) logger.info(f"Uploading assets for git ref: {GITHUB_REF} sha: {GITHUB_SHA}") # Upload YAML template - template_name = "drift-service-catalog.yml" + template_name = "cloudformation/drift-service-catalog.yml" object_key = f"{BUCKET_PREFIX}{template_name}" upload_file(template_name, BUCKET_NAME, object_key, "application/x-yaml") # Upload assets diff --git a/lambda/build/lambda_pipeline_change.py b/lambda/lambda_pipeline_change.py similarity index 100% rename from lambda/build/lambda_pipeline_change.py rename to lambda/lambda_pipeline_change.py diff --git a/lambda/build/lambda_start_pipeline.py b/lambda/lambda_start_pipeline.py similarity index 100% rename from lambda/build/lambda_start_pipeline.py rename to lambda/lambda_start_pipeline.py