From 8b80a77847ee182b0cac53a0421d02b52af9e092 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 15 May 2023 12:08:03 +0200 Subject: [PATCH] DO NOT MERGE Test aws integration on localstack API --- .../pipeline.trigger.integration.tests.sh | 48 ++--- Makefile | 2 +- .../_static/localstack_deployer.yml | 18 ++ .../_static/terraform_deployer_run.sh | 1 + .../system/servicedeployer/terraform.go | 18 ++ .../ec2_metrics/_dev/deploy/tf/cloudwatch.tf | 173 ++++++++++++++++++ .../ec2_metrics/_dev/deploy/tf/env.yml | 4 - .../ec2_metrics/_dev/deploy/tf/main.tf | 118 ++++++++++-- .../ec2_metrics/_dev/deploy/tf/variables.tf | 6 +- .../_dev/test/system/test-default-config.yml | 4 +- 10 files changed, 338 insertions(+), 54 deletions(-) create mode 100644 internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml create mode 100644 test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 157bbd30eb..de6f554f98 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -18,16 +18,16 @@ STACK_COMMAND_TESTS=( test-stack-command-8x ) -for test in ${STACK_COMMAND_TESTS[@]}; do - echo " - label: \":go: Running integration test: ${test}\"" - echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" - echo " agents:" - echo " provider: \"gcp\"" - echo " artifact_paths:" - echo " - build/elastic-stack-dump/stack/*/logs/*.log" - echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" - echo " - build/elastic-stack-status/*/*" -done +# for test in ${STACK_COMMAND_TESTS[@]}; do +# echo " - label: \":go: Running integration test: ${test}\"" +# echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" +# echo " agents:" +# echo " provider: \"gcp\"" +# echo " artifact_paths:" +# echo " - build/elastic-stack-dump/stack/*/logs/*.log" +# echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" +# echo " - build/elastic-stack-status/*/*" +# done CHECK_PACKAGES_TESTS=( test-check-packages-other @@ -35,20 +35,20 @@ CHECK_PACKAGES_TESTS=( test-check-packages-with-custom-agent test-check-packages-benchmarks ) -for test in ${CHECK_PACKAGES_TESTS[@]}; do - echo " - label: \":go: Running integration test: ${test}\"" - echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" - echo " agents:" - echo " provider: \"gcp\"" - echo " artifact_paths:" - echo " - build/test-results/*.xml" - echo " - build/elastic-stack-dump/stack/check-*/logs/*.log" - echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*" - echo " - build/elastic-stack-status/*/*" - if [[ $test =~ with-kind$ ]]; then - echo " - build/kubectl-dump.txt" - fi -done +# for test in ${CHECK_PACKAGES_TESTS[@]}; do +# echo " - label: \":go: Running integration test: ${test}\"" +# echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" +# echo " agents:" +# echo " provider: \"gcp\"" +# echo " artifact_paths:" +# echo " - build/test-results/*.xml" +# echo " - build/elastic-stack-dump/stack/check-*/logs/*.log" +# echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*" +# echo " - build/elastic-stack-status/*/*" +# if [[ $test =~ with-kind$ ]]; then +# echo " - build/kubectl-dump.txt" +# fi +# done pushd test/packages/parallel > /dev/null for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do diff --git a/Makefile b/Makefile index 9c7f7dc126..41c850f392 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-check-packages-benchmarks: PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh test-check-packages-parallel: - PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh + PACKAGE_TEST_TYPE=parallel PACKAGE_UNDER_TEST=aws ./scripts/test-check-packages.sh test-check-packages-with-custom-agent: PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh diff --git a/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml new file mode 100644 index 0000000000..052452f6fc --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml @@ -0,0 +1,18 @@ +version: "3.8" +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: localstack/localstack + hostname: localstack + ports: + - "4566:4566" # LocalStack Gateway + environment: + - SERVICES=sqs,sns + - DEBUG=1 + - DOCKER_HOST=unix:///var/run/docker.sock + - HOST_TMP_FOLDER=${TMPDIR} + - HOSTNAME_EXTERNAL=localstack + - S3_HOSTNAME=localstack + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index 2085bb078f..1a1f56d461 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -21,6 +21,7 @@ trap cleanup EXIT INT TERM terraform init terraform plan terraform apply -auto-approve && touch /tmp/tf-applied +terraform output -json > ./infrastructure.json echo "Terraform definitions applied." diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 4ae68e2b3b..9c80f51e7b 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -24,10 +24,14 @@ import ( const ( terraformDeployerDir = "terraform" terraformDeployerYml = "terraform-deployer.yml" + localstackDeployerYml = "localstack-deployer.yml" terraformDeployerDockerfile = "Dockerfile" terraformDeployerRun = "run.sh" ) +//go:embed _static/localstack_deployer.yml +var localstackDeployerYmlContent string + //go:embed _static/terraform_deployer.yml var terraformDeployerYmlContent string @@ -59,12 +63,21 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic } ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)} + + localstackYmlPath := filepath.Join(configDir, localstackDeployerYml) + _, err = os.Stat(localstackYmlPath) + if err == nil { + ymlPaths = append(ymlPaths, localstackYmlPath) + } + envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) _, err = os.Stat(envYmlPath) if err == nil { ymlPaths = append(ymlPaths, envYmlPath) } + logger.Debug("Print the yml Paths %s", ymlPaths) + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) service := dockerComposeDeployedService{ @@ -130,6 +143,11 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir) resources := []resource.Resource{ + &resource.File{ + Path: localstackDeployerYml, + Content: resource.FileContentLiteral(localstackDeployerYmlContent), + CreateParent: true, + }, &resource.File{ Path: terraformDeployerYml, Content: resource.FileContentLiteral(terraformDeployerYmlContent), diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf new file mode 100644 index 0000000000..49670b1563 --- /dev/null +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf @@ -0,0 +1,173 @@ +resource "aws_cloudwatch_metric_stream" "main" { + name = "my-metric-stream" + role_arn = aws_iam_role.metric_stream_to_firehose.arn + firehose_arn = aws_kinesis_firehose_delivery_stream.s3_stream.arn + output_format = "json" + + include_filter { + namespace = "AWS/EC2" + metric_names = ["CPUUtilization", "NetworkOut"] + } + + include_filter { + namespace = "AWS/EBS" + metric_names = [] + } +} + +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html +data "aws_iam_policy_document" "streams_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["streams.metrics.cloudwatch.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + "iam:passRole", + "cloudwatch:PutMetricStream" + ] + } +} + +resource "aws_iam_role" "metric_stream_to_firehose" { + name = "metric_stream_to_firehose_role" + assume_role_policy = data.aws_iam_policy_document.streams_assume_role.json +} + +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html +data "aws_iam_policy_document" "metric_stream_to_firehose" { + statement { + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.s3_stream.arn] + } +} +resource "aws_iam_role_policy" "metric_stream_to_firehose" { + name = "default" + role = aws_iam_role.metric_stream_to_firehose.id + policy = data.aws_iam_policy_document.metric_stream_to_firehose.json +} + +resource "aws_s3_bucket" "bucket" { + bucket = "metric-stream-test-bucket" +} + +resource "aws_s3_bucket_acl" "bucket_acl" { + bucket = aws_s3_bucket.bucket.id + acl = "private" +} + +data "aws_iam_policy_document" "firehose_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + "iam:passRole", + "cloudwatch:PutMetricStream" + ] + } +} + +resource "aws_iam_role" "firehose_to_s3" { + assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json +} + +data "aws_iam_policy_document" "firehose_to_s3" { + statement { + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + aws_s3_bucket.bucket.arn, + "${aws_s3_bucket.bucket.arn}/*", + ] + } +} + +resource "aws_iam_role_policy" "firehose_to_s3" { + name = "default" + role = aws_iam_role.firehose_to_s3.id + policy = data.aws_iam_policy_document.firehose_to_s3.json +} + +resource "aws_kinesis_firehose_delivery_stream" "s3_stream" { + name = "metric-stream-test-stream" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose_to_s3.arn + bucket_arn = aws_s3_bucket.bucket.arn + } +} + +resource "aws_iam_user" "ecs_deployer" { + name = "ecs_deployer" + path = "*" +} + +# The most important part is the iam:PassRole. With that, this user can give roles to ECS tasks. +# In theory the user can give the task Admin rights. To make sure that does not happen we restrict +# the user and allow him only to hand out roles in /ecs/ path. You still need to be careful not +# to have any roles in there with full admin rights, but no ECS task should have these rights! +resource "aws_iam_user_policy" "ecs_deployer_policy" { + name = "ecs_deployer_policy" + user = aws_iam_user.ecs_deployer.name + policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "ecs:RegisterTaskDefinition", + "ecs:DescribeTaskDefinitions", + "ecs:ListTaskDefinitions", + "ecs:CreateService", + "ecs:UpdateService", + "ecs:DescribeServices", + "ecs:ListServices" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "cloudwatch:PutMetricStream" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : ["iam:PassRole"], + "Resource" : "*" + } + ] + }) +} + +resource "aws_iam_access_key" "ecs_deployer" { + user = aws_iam_user.ecs_deployer.name +} \ No newline at end of file diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml index b795fcdeb2..3e959e419c 100644 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml @@ -2,8 +2,4 @@ version: '2.3' services: terraform: environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - - AWS_PROFILE=${AWS_PROFILE} - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index 00f3ba0c25..6f7b0482f4 100644 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -3,31 +3,109 @@ variable "TEST_RUN_ID" { } provider "aws" { - default_tags { - tags = { - environment = var.ENVIRONMENT - repo = var.REPO - branch = var.BRANCH - build = var.BUILD_ID - created_date = var.CREATED_DATE - } + access_key = "test" + secret_key = "test" + region = "us-east-1" + s3_use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + + endpoints { + apigateway = "http://localstack:4566" + apigatewayv2 = "http://localstack:4566" + cloudformation = "http://localstack:4566" + cloudwatch = "http://localstack:4566" + dynamodb = "http://localstack:4566" + ec2 = "http://localstack:4566" + es = "http://localstack:4566" + elasticache = "http://localstack:4566" + firehose = "http://localstack:4566" + iam = "http://localstack:4566" + kinesis = "http://localstack:4566" + lambda = "http://localstack:4566" + rds = "http://localstack:4566" + redshift = "http://localstack:4566" + route53 = "http://localstack:4566" + s3 = "http://localstack:4566" + secretsmanager = "http://localstack:4566" + ses = "http://localstack:4566" + sns = "http://localstack:4566" + sqs = "http://localstack:4566" + ssm = "http://localstack:4566" + stepfunctions = "http://localstack:4566" + sts = "http://localstack:4566" } } -resource "aws_instance" "i" { - ami = data.aws_ami.latest-amzn.id - monitoring = true - instance_type = "t1.micro" +resource "aws_instance" "elastic_package_aws_test_ec2" { + ami = "ami-0d57c0143330e1fa7" + instance_type = "t2.micro" + tags = { - Name = "elastic-package-test-${var.TEST_RUN_ID}" + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE } } -data "aws_ami" "latest-amzn" { - most_recent = true - owners = [ "amazon" ] # AWS - filter { - name = "name" - values = ["amzn2-ami-minimal-hvm-*-ebs"] - } +resource "aws_cloudwatch_metric_alarm" "foobar" { + alarm_name = "terraform-test-foobar5" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 2 + metric_name = "CPUUtilization" + namespace = "AWS/EC2" + period = 60 + statistic = "Average" + threshold = 1 + alarm_description = "This metric monitors ec2 cpu utilization" + insufficient_data_actions = [] +} + +resource "aws_cloudwatch_dashboard" "EC2_Dashboard" { + dashboard_name = "EC2-Dashboard" + dashboard_body = <