diff --git a/webapp/pom.xml b/webapp/pom.xml index 5aead3203e..93e45627c5 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -192,6 +192,12 @@ 1.12.649 + + com.google.cloud + google-cloud-storage + 2.38.0 + + org.hsqldb hsqldb diff --git a/webapp/src/main/java/com/box/l10n/mojito/gcs/GCSConfiguration.java b/webapp/src/main/java/com/box/l10n/mojito/gcs/GCSConfiguration.java new file mode 100644 index 0000000000..e8d22dbfbe --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/gcs/GCSConfiguration.java @@ -0,0 +1,22 @@ +package com.box.l10n.mojito.gcs; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration for the Google Cloud Storage client. + * + *

Creates a {@link Storage} client using Application Default Credentials (ADC). + */ +@Configuration +@ConditionalOnProperty("l10n.gcs.enabled") +public class GCSConfiguration { + + @Bean + public Storage gcsStorage() { + return StorageOptions.getDefaultInstance().getService(); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/BlobStorageConfiguration.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/BlobStorageConfiguration.java index 20adc91745..e836d415fb 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/BlobStorageConfiguration.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/BlobStorageConfiguration.java @@ -6,8 +6,11 @@ import com.box.l10n.mojito.service.blobstorage.database.DatabaseBlobStorageCleanupJob; import com.box.l10n.mojito.service.blobstorage.database.DatabaseBlobStorageConfigurationProperties; import com.box.l10n.mojito.service.blobstorage.database.MBlobRepository; +import com.box.l10n.mojito.service.blobstorage.gcs.GCSBlobStorage; +import com.box.l10n.mojito.service.blobstorage.gcs.GCSBlobStorageConfigurationProperties; import com.box.l10n.mojito.service.blobstorage.s3.S3BlobStorage; import com.box.l10n.mojito.service.blobstorage.s3.S3BlobStorageConfigurationProperties; +import com.google.cloud.storage.Storage; import java.time.Duration; import org.quartz.JobDetail; import org.quartz.SimpleTrigger; @@ -15,6 +18,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -28,8 +32,14 @@ *

{@link DatabaseBlobStorage} is the default implementation but it should be use only for * testing or deployments with limited load. * - *

Consider using {@link S3BlobStorage} for larger deployment. An {@link AmazonS3} client must be - * configured first, and then the storage enabled with the `l10n.blob-storage.type=s3` property + *

Consider using one of the following alternatives for larger deployment: + * + *

+ * + *

The `l10n.blob-storage.type` property can be set switch between the desired implementation. */ @Configuration public class BlobStorageConfiguration { @@ -51,6 +61,20 @@ public S3BlobStorage s3BlobStorage() { } } + @ConditionalOnProperty(value = "l10n.blob-storage.type", havingValue = "gcs") + @ConditionalOnBean(Storage.class) + @Configuration + static class GCSBlobStorageConfiguration { + + @Autowired GCSBlobStorageConfigurationProperties gcsBlobStorageConfigurationProperties; + + @Bean + public GCSBlobStorage gcsBlobStorage(Storage gcsStorage) { + logger.info("Configure GCSBlobStorage (using Application Default Credentials)"); + return new GCSBlobStorage(gcsStorage, gcsBlobStorageConfigurationProperties); + } + } + @ConditionalOnProperty( value = "l10n.blob-storage.type", havingValue = "database", diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorage.java new file mode 100644 index 0000000000..3ab770bf4a --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorage.java @@ -0,0 +1,101 @@ +package com.box.l10n.mojito.service.blobstorage.gcs; + +import com.box.l10n.mojito.service.blobstorage.BlobStorage; +import com.box.l10n.mojito.service.blobstorage.Retention; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.base.Preconditions; +import java.time.OffsetDateTime; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation that uses Google Cloud Storage to store blobs. + * + *

To use it, first enable the client through {@link com.box.l10n.mojito.gcs.GCSConfiguration}. + * + *

Rely on GCS lifecyle rules to clean up expired blobs. This must be set up on the bucket, + * otherwise no cleanup will happen. + * + *

Objects will have a Custom-Time metadata + * field set to the end of the desired retention period (except {@link Retention#PERMANENT} where + * this is not set). + * + *

On the bucket, configure a single lifecycle rule: + * + *

+ * + * That will delete objects once the current time is past their Custom-Time (end of retention). Note + * that objects without Custom-Time set (i.e. {@link Retention#PERMANENT}) are never deleted by this + * rule. + * + *

See Object + * Lifecycle Management for reference. + */ +public class GCSBlobStorage implements BlobStorage { + + static final Logger logger = LoggerFactory.getLogger(GCSBlobStorage.class); + + private final Storage storage; + private final GCSBlobStorageConfigurationProperties configurationProperties; + + private static final String byteContentType = "application/octet-stream"; + + public GCSBlobStorage( + Storage storage, GCSBlobStorageConfigurationProperties configurationProperties) { + Preconditions.checkNotNull(storage); + Preconditions.checkNotNull(configurationProperties); + this.storage = storage; + this.configurationProperties = configurationProperties; + } + + public Optional getBytes(String name) { + Blob blob = storage.get(BlobId.of(configurationProperties.getBucket(), getFullName(name))); + if (blob == null) { + return Optional.empty(); + } + return Optional.of(blob.getContent()); + } + + public void put(String name, byte[] content, Retention retention) { + BlobInfo.Builder builder = + BlobInfo.newBuilder(BlobId.of(configurationProperties.getBucket(), getFullName(name))) + .setContentType(byteContentType); + + customTimeAtEndOfRetention(retention).ifPresent(builder::setCustomTimeOffsetDateTime); + + storage.create(builder.build(), content); + } + + public void delete(String name) { + storage.delete(BlobId.of(configurationProperties.getBucket(), getFullName(name))); + } + + public boolean exists(String name) { + Blob blob = storage.get(BlobId.of(configurationProperties.getBucket(), getFullName(name))); + return blob != null; + } + + /** + * GCS Custom-Time at end of retention for lifecycle (daysSinceCustomTime: 0). Empty means no + * Custom-Time (object never expires). Exhaustive on {@link Retention} so new enum values require + * an explicit case here. + */ + private static Optional customTimeAtEndOfRetention(Retention retention) { + return switch (retention) { + case PERMANENT -> Optional.empty(); + case MIN_1_DAY -> Optional.of(OffsetDateTime.now().plusDays(1)); + }; + } + + String getFullName(String name) { + return configurationProperties.getPrefix() + "/" + name; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageConfigurationProperties.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageConfigurationProperties.java new file mode 100644 index 0000000000..09202c82fa --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageConfigurationProperties.java @@ -0,0 +1,28 @@ +package com.box.l10n.mojito.service.blobstorage.gcs; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.blob-storage.gcs") +public class GCSBlobStorageConfigurationProperties { + + String bucket = "mojito"; + String prefix = "mojito"; + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/s3/S3BlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/s3/S3BlobStorage.java index 41587423b1..2d053fe293 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/s3/S3BlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/s3/S3BlobStorage.java @@ -25,6 +25,9 @@ /** * Implementation that uses S3 to store blobs. * + *

To use it, first enable and configure the client through {@link + * com.box.l10n.mojito.aws.s3.AmazonS3Configuration}. + * *

Rely on S3 lifecyle rules to cleanup expired blobs. This must be setup manually else no clean * up will happen. * diff --git a/webapp/src/main/resources/config/application.properties b/webapp/src/main/resources/config/application.properties index 34ece931ef..dc766d0ce0 100644 --- a/webapp/src/main/resources/config/application.properties +++ b/webapp/src/main/resources/config/application.properties @@ -296,4 +296,10 @@ spring.session.jdbc.cleanup-cron=- #l10n.blob-storage.s3.bucket=mojito #l10n.blob-storage.s3.prefix=mojito +# GCS implementation (uses Application Default Credentials) +#l10n.gcs.enabled=true +#l10n.blob-storage.type=gcs +#l10n.blob-storage.gcs.bucket=mojito +#l10n.blob-storage.gcs.prefix=mojito + diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageTest.java new file mode 100644 index 0000000000..a6f313a182 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageTest.java @@ -0,0 +1,115 @@ +package com.box.l10n.mojito.service.blobstorage.gcs; + +import com.box.l10n.mojito.gcs.GCSConfiguration; +import com.box.l10n.mojito.service.blobstorage.BlobStorage; +import com.box.l10n.mojito.service.blobstorage.BlobStorageTestShared; +import com.google.cloud.storage.Storage; +import java.util.UUID; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest( + classes = { + GCSBlobStorageTest.class, + GCSConfiguration.class, + GCSBlobStorageConfigurationProperties.class, + GCSBlobStorageTest.TestConfig.class, + }) +@EnableConfigurationProperties +@TestPropertySource(properties = {"l10n.gcs.enabled=true", "l10n.blob-storage.type=gcs"}) +public class GCSBlobStorageTest implements BlobStorageTestShared { + + @Autowired(required = false) + GCSBlobStorage gcsBlobStorage; + + @Override + public BlobStorage getBlobStorage() { + return gcsBlobStorage; + } + + @Before + @Override + public void bbefore() { + BlobStorageTestShared.super.bbefore(); + // Skip tests when GCS ADC or bucket is not configured + try { + getBlobStorage().exists("probe-" + UUID.randomUUID()); + } catch (Exception e) { + Assume.assumeNoException("GCS credentials/ADC not available", e); + } + } + + @Test + @Override + public void testNoMatchString() { + BlobStorageTestShared.super.testNoMatchString(); + } + + @Test + @Override + public void testNoMatchBytes() { + BlobStorageTestShared.super.testNoMatchBytes(); + } + + @Test + @Override + public void testMatchString() { + BlobStorageTestShared.super.testMatchString(); + } + + @Test + @Override + public void testMatchBytes() { + BlobStorageTestShared.super.testMatchBytes(); + } + + @Test + @Override + public void testMatchMin1DayRetentionString() { + BlobStorageTestShared.super.testMatchMin1DayRetentionString(); + } + + @Test + @Override + public void testMatchMin1DayRetentionBytes() { + BlobStorageTestShared.super.testMatchMin1DayRetentionBytes(); + } + + @Test + @Override + public void testUpdatesWithPut() { + BlobStorageTestShared.super.testUpdatesWithPut(); + } + + @Test + @Override + public void testExsits() { + BlobStorageTestShared.super.testExsits(); + } + + @Test + @Override + public void testDelete() { + BlobStorageTestShared.super.testDelete(); + } + + @Configuration + static class TestConfig { + + @Bean + public GCSBlobStorage gcsBlobStorage( + Storage gcsStorage, GCSBlobStorageConfigurationProperties configurationProperties) { + return new GCSBlobStorage(gcsStorage, configurationProperties); + } + } +}