diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index d6477e05b15d5..b29bb7a8cd3b7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -142,6 +142,8 @@ class ClusterConfiguration { // there are cases when value depends on task that is not executed yet on configuration stage Map systemProperties = new HashMap<>() + Map environmentVariables = new HashMap<>() + Map settings = new HashMap<>() Map keystoreSettings = new HashMap<>() @@ -164,6 +166,11 @@ class ClusterConfiguration { systemProperties.put(property, value) } + @Input + void environment(String variable, Object value) { + environmentVariables.put(variable, value) + } + @Input void setting(String name, Object value) { settings.put(name, value) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 7844ea77fc18f..0dd56b863324f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -181,6 +181,7 @@ class NodeInfo { args.addAll("-E", "node.portsfile=true") env = [:] + env.putAll(config.environmentVariables) for (Map.Entry property : System.properties.entrySet()) { if (property.key.startsWith('tests.es.')) { args.add("-E") diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 0d73e35f18ec3..19ead367204ba 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -13,8 +13,8 @@ include::install_remove.asciidoc[] ==== Getting started with AWS The plugin provides a repository type named `s3` which may be used when creating a repository. -The repository defaults to using -http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[IAM Role] +The repository defaults to using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html[ECS IAM Role] or +http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[EC2 IAM Role] credentials for authentication. The only mandatory setting is the bucket name: [source,js] diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 225d523817e7d..181891e20564d 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -92,11 +92,15 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") +String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs") +String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") + // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath - && !s3EC2Bucket && !s3EC2BasePath) { + && !s3EC2Bucket && !s3EC2BasePath + && !s3ECSBucket && !s3ECSBasePath) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' s3PermanentBucket = 'permanent-bucket-test' @@ -105,10 +109,14 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3EC2Bucket = 'ec2-bucket-test' s3EC2BasePath = 'integration_test' + s3ECSBucket = 'ecs-bucket-test' + s3ECSBasePath = 'integration_test' + useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath - || !s3EC2Bucket || !s3EC2BasePath) { + || !s3EC2Bucket || !s3EC2BasePath + || !s3ECSBucket || !s3ECSBasePath) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -284,7 +292,8 @@ if (useFixture && minioDistribution) { // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 integTestMinioRunner.systemProperty 'tests.rest.blacklist', [ 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*' + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/50_repository_ecs_credentials/*' ].join(",") project.check.dependsOn(integTestMinio) @@ -302,7 +311,8 @@ task s3FixtureProperties { "s3Fixture.temporary_bucket_name" : s3TemporaryBucket, "s3Fixture.temporary_key" : s3TemporaryAccessKey, "s3Fixture.temporary_session_token": s3TemporarySessionToken, - "s3Fixture.ec2_bucket_name" : s3EC2Bucket + "s3Fixture.ec2_bucket_name" : s3EC2Bucket, + "s3Fixture.ecs_bucket_name" : s3ECSBucket ] doLast { @@ -327,7 +337,9 @@ Map expansions = [ 'temporary_bucket': s3TemporaryBucket, 'temporary_base_path': s3TemporaryBasePath, 'ec2_bucket': s3EC2Bucket, - 'ec2_base_path': s3EC2BasePath + 'ec2_base_path': s3EC2BasePath, + 'ecs_bucket': s3ECSBucket, + 'ecs_base_path': s3ECSBasePath ] processTestResources { @@ -364,6 +376,34 @@ integTestCluster { } } +integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*' + +/// +RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) { + description = "Runs tests using the ECS repository." +} + +// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: +project.afterEvaluate { + ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration + cluster.dependsOn(project.s3Fixture) + + cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}" + + Task integTestECSTask = project.tasks.getByName('integTestECS') + integTestECSTask.clusterConfig.plugin(project.path) + integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', + "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint" + integTestECSRunner.systemProperty 'tests.rest.blacklist', [ + 'repository_s3/10_basic/*', + 'repository_s3/20_repository_permanent_credentials/*', + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*' + ].join(",") +} +project.check.dependsOn(integTestECS) +/// + thirdPartyAudit.excludes = [ // classes are missing 'javax.servlet.ServletContextEvent', diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 91a7a30024b78..b177686bd71a6 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -22,7 +22,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -156,10 +156,11 @@ protected synchronized void releaseCachedClients() { } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { - private final InstanceProfileCredentialsProvider credentials; + private final AWSCredentialsProvider credentials; private PrivilegedInstanceProfileCredentialsProvider() { - this.credentials = new InstanceProfileCredentialsProvider(); + // InstanceProfileCredentialsProvider as last item of chain + this.credentials = new EC2ContainerCredentialsProviderWrapper(); } @Override diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index ce6c472314999..a411a1c53cf36 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -88,7 +88,10 @@ private AmazonS3Fixture(final String workingDir, Properties properties) { final Bucket ec2Bucket = new Bucket("s3Fixture.ec2", randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10)); - this.handlers = defaultHandlers(buckets, ec2Bucket); + final Bucket ecsBucket = new Bucket("s3Fixture.ecs", + randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10)); + + this.handlers = defaultHandlers(buckets, ec2Bucket, ecsBucket); } private static String nonAuthPath(Request request) { @@ -174,7 +177,7 @@ public static void main(final String[] args) throws Exception { } /** Builds the default request handlers **/ - private PathTrie defaultHandlers(final Map buckets, final Bucket ec2Bucket) { + private PathTrie defaultHandlers(final Map buckets, final Bucket ec2Bucket, final Bucket ecsBucket) { final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); // HEAD Object @@ -400,11 +403,18 @@ private PathTrie defaultHandlers(final Map bucke handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> { final String profileName = request.getParam("profileName"); if (EC2_PROFILE.equals(profileName) == false) { - return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8)); + return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown profile".getBytes(UTF_8)); } return credentialResponseFunction.apply(profileName, ec2Bucket.key, ec2Bucket.token); }); + // GET + // + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/ecs_credentials_endpoint"), + (request) -> credentialResponseFunction.apply("CPV_ECS", ecsBucket.key, ecsBucket.token)); + + return handlers; } diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml new file mode 100644 index 0000000000000..54929e6e3ad82 --- /dev/null +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -0,0 +1,243 @@ +# Integration tests for repository-s3 + +--- +setup: + + # Register repository with ecs credentials + - do: + snapshot.create_repository: + repository: repository_ecs + body: + type: s3 + settings: + bucket: ${ecs_bucket} + client: integration_test_ecs + base_path: ${ecs_base_path} + canned_acl: private + storage_class: standard + +--- +"Snapshot and Restore with repository-s3 using ecs credentials": + + # Get repository + - do: + snapshot.get_repository: + repository: repository_ecs + + - match: { repository_ecs.settings.bucket : ${ecs_bucket} } + - match: { repository_ecs.settings.client : "integration_test_ecs" } + - match: { repository_ecs.settings.base_path : ${ecs_base_path} } + - match: { repository_ecs.settings.canned_acl : "private" } + - match: { repository_ecs.settings.storage_class : "standard" } + - is_false: repository_ecs.settings.access_key + - is_false: repository_ecs.settings.secret_key + - is_false: repository_ecs.settings.session_token + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_ecs + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_ecs + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_ecs + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_ecs + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_ecs + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_ecs + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_ecs + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_ecs + snapshot: snapshot-one + +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_ecs + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test_temporary + +--- +"Register a repository with a non existing client": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_ecs + body: + type: s3 + settings: + bucket: repository_ecs + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_ecs + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_ecs + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_ecs + snapshot: missing + wait_for_completion: true + +--- +teardown: + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_ecs