diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index c232b6616b..035e567f21 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -18,16 +18,16 @@ STACK_COMMAND_TESTS=( test-stack-command-8x ) -for test in ${STACK_COMMAND_TESTS[@]}; do - echo " - label: \":go: Running integration test: ${test}\"" - echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" - echo " agents:" - echo " provider: \"gcp\"" - echo " artifact_paths:" - echo " - build/elastic-stack-dump/stack/*/logs/*.log" - echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" - echo " - build/elastic-stack-status/*/*" -done +# for test in ${STACK_COMMAND_TESTS[@]}; do +# echo " - label: \":go: Running integration test: ${test}\"" +# echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" +# echo " agents:" +# echo " provider: \"gcp\"" +# echo " artifact_paths:" +# echo " - build/elastic-stack-dump/stack/*/logs/*.log" +# echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" +# echo " - build/elastic-stack-status/*/*" +# done CHECK_PACKAGES_TESTS=( test-check-packages-other @@ -35,6 +35,7 @@ CHECK_PACKAGES_TESTS=( test-check-packages-with-custom-agent test-check-packages-benchmarks ) + for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " - label: \":go: Running integration test: ${test}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" diff --git a/Makefile b/Makefile index 21b4fe28ec..4fe2ff7d98 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-check-packages-benchmarks: PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh test-check-packages-parallel: - PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh + PACKAGE_TEST_TYPE=parallel PACKAGE_UNDER_TEST=aws ./scripts/test-check-packages.sh test-check-packages-with-custom-agent: PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh diff --git a/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml new file mode 100644 index 0000000000..052452f6fc --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml @@ -0,0 +1,18 @@ +version: "3.8" +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: localstack/localstack + hostname: localstack + ports: + - "4566:4566" # LocalStack Gateway + environment: + - SERVICES=sqs,sns + - DEBUG=1 + - DOCKER_HOST=unix:///var/run/docker.sock + - HOST_TMP_FOLDER=${TMPDIR} + - HOSTNAME_EXTERNAL=localstack + - S3_HOSTNAME=localstack + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index 80054b46c2..6b54f6e334 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -20,6 +20,7 @@ trap cleanup EXIT INT TERM terraform init terraform plan + terraform apply -auto-approve terraform output -json > /output/tfOutputValues.json diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 1a61125224..c88bae9b08 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -23,12 +23,16 @@ import ( const ( terraformDeployerDir = "terraform" terraformDeployerYml = "terraform-deployer.yml" + localstackDeployerYml = "localstack-deployer.yml" terraformDeployerDockerfile = "Dockerfile" terraformDeployerRun = "run.sh" terraformOutputPrefix = "TF_OUTPUT_" terraformOutputJsonFile = "tfOutputValues.json" ) +//go:embed _static/localstack_deployer.yml +var localstackDeployerYmlContent string + //go:embed _static/terraform_deployer.yml var terraformDeployerYmlContent string @@ -97,12 +101,21 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic } ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)} + + localstackYmlPath := filepath.Join(configDir, localstackDeployerYml) + _, err = os.Stat(localstackYmlPath) + if err == nil { + ymlPaths = append(ymlPaths, localstackYmlPath) + } + envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) _, err = os.Stat(envYmlPath) if err == nil { ymlPaths = append(ymlPaths, envYmlPath) } + logger.Debug("Print the yml Paths %s", ymlPaths) + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) service := dockerComposeDeployedService{ @@ -174,6 +187,11 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir) resources := []resource.Resource{ + &resource.File{ + Path: localstackDeployerYml, + Content: resource.FileContentLiteral(localstackDeployerYmlContent), + CreateParent: true, + }, &resource.File{ Path: terraformDeployerYml, Content: resource.FileContentLiteral(terraformDeployerYmlContent), diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf new file mode 100644 index 0000000000..49670b1563 --- /dev/null +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf @@ -0,0 +1,173 @@ +resource "aws_cloudwatch_metric_stream" "main" { + name = "my-metric-stream" + role_arn = aws_iam_role.metric_stream_to_firehose.arn + firehose_arn = aws_kinesis_firehose_delivery_stream.s3_stream.arn + output_format = "json" + + include_filter { + namespace = "AWS/EC2" + metric_names = ["CPUUtilization", "NetworkOut"] + } + + include_filter { + namespace = "AWS/EBS" + metric_names = [] + } +} + +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html +data "aws_iam_policy_document" "streams_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["streams.metrics.cloudwatch.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + "iam:passRole", + "cloudwatch:PutMetricStream" + ] + } +} + +resource "aws_iam_role" "metric_stream_to_firehose" { + name = "metric_stream_to_firehose_role" + assume_role_policy = data.aws_iam_policy_document.streams_assume_role.json +} + +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html +data "aws_iam_policy_document" "metric_stream_to_firehose" { + statement { + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.s3_stream.arn] + } +} +resource "aws_iam_role_policy" "metric_stream_to_firehose" { + name = "default" + role = aws_iam_role.metric_stream_to_firehose.id + policy = data.aws_iam_policy_document.metric_stream_to_firehose.json +} + +resource "aws_s3_bucket" "bucket" { + bucket = "metric-stream-test-bucket" +} + +resource "aws_s3_bucket_acl" "bucket_acl" { + bucket = aws_s3_bucket.bucket.id + acl = "private" +} + +data "aws_iam_policy_document" "firehose_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + "iam:passRole", + "cloudwatch:PutMetricStream" + ] + } +} + +resource "aws_iam_role" "firehose_to_s3" { + assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json +} + +data "aws_iam_policy_document" "firehose_to_s3" { + statement { + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + aws_s3_bucket.bucket.arn, + "${aws_s3_bucket.bucket.arn}/*", + ] + } +} + +resource "aws_iam_role_policy" "firehose_to_s3" { + name = "default" + role = aws_iam_role.firehose_to_s3.id + policy = data.aws_iam_policy_document.firehose_to_s3.json +} + +resource "aws_kinesis_firehose_delivery_stream" "s3_stream" { + name = "metric-stream-test-stream" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose_to_s3.arn + bucket_arn = aws_s3_bucket.bucket.arn + } +} + +resource "aws_iam_user" "ecs_deployer" { + name = "ecs_deployer" + path = "*" +} + +# The most important part is the iam:PassRole. With that, this user can give roles to ECS tasks. +# In theory the user can give the task Admin rights. To make sure that does not happen we restrict +# the user and allow him only to hand out roles in /ecs/ path. You still need to be careful not +# to have any roles in there with full admin rights, but no ECS task should have these rights! +resource "aws_iam_user_policy" "ecs_deployer_policy" { + name = "ecs_deployer_policy" + user = aws_iam_user.ecs_deployer.name + policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "ecs:RegisterTaskDefinition", + "ecs:DescribeTaskDefinitions", + "ecs:ListTaskDefinitions", + "ecs:CreateService", + "ecs:UpdateService", + "ecs:DescribeServices", + "ecs:ListServices" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "cloudwatch:PutMetricStream" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : ["iam:PassRole"], + "Resource" : "*" + } + ] + }) +} + +resource "aws_iam_access_key" "ecs_deployer" { + user = aws_iam_user.ecs_deployer.name +} \ No newline at end of file diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml index b795fcdeb2..3e959e419c 100644 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml @@ -2,8 +2,4 @@ version: '2.3' services: terraform: environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - - AWS_PROFILE=${AWS_PROFILE} - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index 32738ec4d4..c8789389ce 100644 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -3,35 +3,37 @@ variable "TEST_RUN_ID" { } provider "aws" { - default_tags { - tags = { - environment = var.ENVIRONMENT - repo = var.REPO - branch = var.BRANCH - build = var.BUILD_ID - created_date = var.CREATED_DATE - } - } -} + access_key = "test" + secret_key = "test" + region = "us-east-1" + s3_use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true -resource "aws_instance" "i" { - ami = data.aws_ami.latest-amzn.id - monitoring = true - instance_type = "t1.micro" - tags = { - Name = "elastic-package-test-${var.TEST_RUN_ID}" + endpoints { + apigateway = "http://localstack:4566" + apigatewayv2 = "http://localstack:4566" + cloudformation = "http://localstack:4566" + cloudwatch = "http://localstack:4566" + dynamodb = "http://localstack:4566" + ec2 = "http://localstack:4566" + es = "http://localstack:4566" + elasticache = "http://localstack:4566" + firehose = "http://localstack:4566" + iam = "http://localstack:4566" + kinesis = "http://localstack:4566" + lambda = "http://localstack:4566" + rds = "http://localstack:4566" + redshift = "http://localstack:4566" + route53 = "http://localstack:4566" + s3 = "http://localstack:4566" + secretsmanager = "http://localstack:4566" + ses = "http://localstack:4566" + sns = "http://localstack:4566" + sqs = "http://localstack:4566" + ssm = "http://localstack:4566" + stepfunctions = "http://localstack:4566" + sts = "http://localstack:4566" } -} - -data "aws_ami" "latest-amzn" { - most_recent = true - owners = [ "amazon" ] # AWS - filter { - name = "name" - values = ["amzn2-ami-minimal-hvm-*-ebs"] - } -} - -output "instance_id" { - value = aws_instance.i.id } \ No newline at end of file