Implement Google Cloud Storage-based blob storage

webapp/pom.xml

@@ -192,6 +192,12 @@
             <version>1.12.649</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>2.38.0</version>
+        </dependency>
+
         <dependency>
             <groupId>org.hsqldb</groupId>
             <artifactId>hsqldb</artifactId>

webapp/src/main/java/com/box/l10n/mojito/gcs/GCSConfiguration.java

@@ -0,0 +1,22 @@
+package com.box.l10n.mojito.gcs;
+
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Configuration for the Google Cloud Storage client.
+ *
+ * <p>Creates a {@link Storage} client using Application Default Credentials (ADC).
+ */
+@Configuration
+@ConditionalOnProperty("l10n.gcs.enabled")
+public class GCSConfiguration {
+
+  @Bean
+  public Storage gcsStorage() {
+    return StorageOptions.getDefaultInstance().getService();
+  }
+}

webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/BlobStorageConfiguration.java

@@ -6,15 +6,19 @@
 import com.box.l10n.mojito.service.blobstorage.database.DatabaseBlobStorageCleanupJob;
 import com.box.l10n.mojito.service.blobstorage.database.DatabaseBlobStorageConfigurationProperties;
 import com.box.l10n.mojito.service.blobstorage.database.MBlobRepository;
+import com.box.l10n.mojito.service.blobstorage.gcs.GCSBlobStorage;
+import com.box.l10n.mojito.service.blobstorage.gcs.GCSBlobStorageConfigurationProperties;
 import com.box.l10n.mojito.service.blobstorage.s3.S3BlobStorage;
 import com.box.l10n.mojito.service.blobstorage.s3.S3BlobStorageConfigurationProperties;
+import com.google.cloud.storage.Storage;
 import java.time.Duration;
 import org.quartz.JobDetail;
 import org.quartz.SimpleTrigger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -28,8 +32,14 @@
  * <p>{@link DatabaseBlobStorage} is the default implementation but it should be use only for
  * testing or deployments with limited load.
  *
- * <p>Consider using {@link S3BlobStorage} for larger deployment. An {@link AmazonS3} client must be
- * configured first, and then the storage enabled with the `l10n.blob-storage.type=s3` property
+ * <p>Consider using one of the following alternatives for larger deployment:
+ *
+ * <ul>
+ *   <li>{@link S3BlobStorage} with {@link AmazonS3} client
+ *   <li>{@link GCSBlobStorage} with Google Cloud {@link Storage} client
+ * </ul>
+ *
+ * <p>The `l10n.blob-storage.type` property can be set switch between the desired implementation.
  */
 @Configuration
 public class BlobStorageConfiguration {
@@ -51,6 +61,20 @@ public S3BlobStorage s3BlobStorage() {
     }
   }
 
+  @ConditionalOnProperty(value = "l10n.blob-storage.type", havingValue = "gcs")
+  @ConditionalOnBean(Storage.class)
+  @Configuration
+  static class GCSBlobStorageConfiguration {
+
+    @Autowired GCSBlobStorageConfigurationProperties gcsBlobStorageConfigurationProperties;
+
+    @Bean
+    public GCSBlobStorage gcsBlobStorage(Storage gcsStorage) {
+      logger.info("Configure GCSBlobStorage (using Application Default Credentials)");
+      return new GCSBlobStorage(gcsStorage, gcsBlobStorageConfigurationProperties);
+    }
+  }
+
   @ConditionalOnProperty(
       value = "l10n.blob-storage.type",
       havingValue = "database",

webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorage.java

@@ -0,0 +1,101 @@
+package com.box.l10n.mojito.service.blobstorage.gcs;
+
+import com.box.l10n.mojito.service.blobstorage.BlobStorage;
+import com.box.l10n.mojito.service.blobstorage.Retention;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.common.base.Preconditions;
+import java.time.OffsetDateTime;
+import java.util.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation that uses Google Cloud Storage to store blobs.
+ *
+ * <p>To use it, first enable the client through {@link com.box.l10n.mojito.gcs.GCSConfiguration}.
+ *
+ * <p>Rely on GCS lifecyle rules to clean up expired blobs. This must be set up on the bucket,
+ * otherwise no cleanup will happen.
+ *
+ * <p>Objects will have a <a
+ * href="https://docs.cloud.google.com/storage/docs/metadata#custom-time">Custom-Time</a> metadata
+ * field set to the end of the desired retention period (except {@link Retention#PERMANENT} where
+ * this is not set).
+ *
+ * <p>On the bucket, configure a single lifecycle rule:
+ *
+ * <ul>
+ *   <li>action: Delete
+ *   <li>condition: {@code daysSinceCustomTime: 0}
+ * </ul>
+ *
+ * That will delete objects once the current time is past their Custom-Time (end of retention). Note
+ * that objects without Custom-Time set (i.e. {@link Retention#PERMANENT}) are never deleted by this
+ * rule.
+ *
+ * <p>See <a href="https://docs.cloud.google.com/storage/docs/lifecycle#dayssincecustomtime">Object
+ * Lifecycle Management</a> for reference.
+ */
+public class GCSBlobStorage implements BlobStorage {
+
+  static final Logger logger = LoggerFactory.getLogger(GCSBlobStorage.class);
+
+  private final Storage storage;
+  private final GCSBlobStorageConfigurationProperties configurationProperties;
+
+  private static final String byteContentType = "application/octet-stream";
+
+  public GCSBlobStorage(
+      Storage storage, GCSBlobStorageConfigurationProperties configurationProperties) {
+    Preconditions.checkNotNull(storage);
+    Preconditions.checkNotNull(configurationProperties);
+    this.storage = storage;
+    this.configurationProperties = configurationProperties;
+  }
+
+  public Optional<byte[]> getBytes(String name) {
+    Blob blob = storage.get(BlobId.of(configurationProperties.getBucket(), getFullName(name)));
+    if (blob == null) {
+      return Optional.empty();
+    }
+    return Optional.of(blob.getContent());
+  }
+
+  public void put(String name, byte[] content, Retention retention) {
+    BlobInfo.Builder builder =
+        BlobInfo.newBuilder(BlobId.of(configurationProperties.getBucket(), getFullName(name)))
+            .setContentType(byteContentType);
+
+    customTimeAtEndOfRetention(retention).ifPresent(builder::setCustomTimeOffsetDateTime);
+
+    storage.create(builder.build(), content);
+  }
+
+  public void delete(String name) {
+    storage.delete(BlobId.of(configurationProperties.getBucket(), getFullName(name)));
+  }
+
+  public boolean exists(String name) {
+    Blob blob = storage.get(BlobId.of(configurationProperties.getBucket(), getFullName(name)));
+    return blob != null;
+  }
+
+  /**
+   * GCS Custom-Time at end of retention for lifecycle (daysSinceCustomTime: 0). Empty means no
+   * Custom-Time (object never expires). Exhaustive on {@link Retention} so new enum values require
+   * an explicit case here.
+   */
+  private static Optional<OffsetDateTime> customTimeAtEndOfRetention(Retention retention) {
+    return switch (retention) {
+      case PERMANENT -> Optional.empty();
+      case MIN_1_DAY -> Optional.of(OffsetDateTime.now().plusDays(1));
+    };
+  }
+
+  String getFullName(String name) {
+    return configurationProperties.getPrefix() + "/" + name;
+  }
+}

webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageConfigurationProperties.java

@@ -0,0 +1,28 @@
+package com.box.l10n.mojito.service.blobstorage.gcs;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties("l10n.blob-storage.gcs")
+public class GCSBlobStorageConfigurationProperties {
+
+  String bucket = "mojito";
+  String prefix = "mojito";
+
+  public String getBucket() {
+    return bucket;
+  }
+
+  public void setBucket(String bucket) {
+    this.bucket = bucket;
+  }
+
+  public String getPrefix() {
+    return prefix;
+  }
+
+  public void setPrefix(String prefix) {
+    this.prefix = prefix;
+  }
+}

webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/s3/S3BlobStorage.java

@@ -25,6 +25,9 @@
 /**
  * Implementation that uses S3 to store blobs.
  *
+ * <p>To use it, first enable and configure the client through {@link
+ * com.box.l10n.mojito.aws.s3.AmazonS3Configuration}.
+ *
  * <p>Rely on S3 lifecyle rules to cleanup expired blobs. This must be setup manually else no clean
  * up will happen.
  *

webapp/src/main/resources/config/application.properties

@@ -296,4 +296,10 @@ spring.session.jdbc.cleanup-cron=-
 #l10n.blob-storage.s3.bucket=mojito
 #l10n.blob-storage.s3.prefix=mojito
 
+# GCS implementation (uses Application Default Credentials)
+#l10n.gcs.enabled=true
+#l10n.blob-storage.type=gcs
+#l10n.blob-storage.gcs.bucket=mojito
+#l10n.blob-storage.gcs.prefix=mojito
+
 

webapp/src/test/java/com/box/l10n/mojito/service/blobstorage/gcs/GCSBlobStorageTest.java

@@ -0,0 +1,115 @@
+package com.box.l10n.mojito.service.blobstorage.gcs;
+
+import com.box.l10n.mojito.gcs.GCSConfiguration;
+import com.box.l10n.mojito.service.blobstorage.BlobStorage;
+import com.box.l10n.mojito.service.blobstorage.BlobStorageTestShared;
+import com.google.cloud.storage.Storage;
+import java.util.UUID;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest(
+    classes = {
+      GCSBlobStorageTest.class,
+      GCSConfiguration.class,
+      GCSBlobStorageConfigurationProperties.class,
+      GCSBlobStorageTest.TestConfig.class,
+    })
+@EnableConfigurationProperties
+@TestPropertySource(properties = {"l10n.gcs.enabled=true", "l10n.blob-storage.type=gcs"})
+public class GCSBlobStorageTest implements BlobStorageTestShared {
+
+  @Autowired(required = false)
+  GCSBlobStorage gcsBlobStorage;
+
+  @Override
+  public BlobStorage getBlobStorage() {
+    return gcsBlobStorage;
+  }
+
+  @Before
+  @Override
+  public void bbefore() {
+    BlobStorageTestShared.super.bbefore();
+    // Skip tests when GCS ADC or bucket is not configured
+    try {
+      getBlobStorage().exists("probe-" + UUID.randomUUID());
+    } catch (Exception e) {
+      Assume.assumeNoException("GCS credentials/ADC not available", e);
+    }
+  }
+
+  @Test
+  @Override
+  public void testNoMatchString() {
+    BlobStorageTestShared.super.testNoMatchString();
+  }
+
+  @Test
+  @Override
+  public void testNoMatchBytes() {
+    BlobStorageTestShared.super.testNoMatchBytes();
+  }
+
+  @Test
+  @Override
+  public void testMatchString() {
+    BlobStorageTestShared.super.testMatchString();
+  }
+
+  @Test
+  @Override
+  public void testMatchBytes() {
+    BlobStorageTestShared.super.testMatchBytes();
+  }
+
+  @Test
+  @Override
+  public void testMatchMin1DayRetentionString() {
+    BlobStorageTestShared.super.testMatchMin1DayRetentionString();
+  }
+
+  @Test
+  @Override
+  public void testMatchMin1DayRetentionBytes() {
+    BlobStorageTestShared.super.testMatchMin1DayRetentionBytes();
+  }
+
+  @Test
+  @Override
+  public void testUpdatesWithPut() {
+    BlobStorageTestShared.super.testUpdatesWithPut();
+  }
+
+  @Test
+  @Override
+  public void testExsits() {
+    BlobStorageTestShared.super.testExsits();
+  }
+
+  @Test
+  @Override
+  public void testDelete() {
+    BlobStorageTestShared.super.testDelete();
+  }
+
+  @Configuration
+  static class TestConfig {
+
+    @Bean
+    public GCSBlobStorage gcsBlobStorage(
+        Storage gcsStorage, GCSBlobStorageConfigurationProperties configurationProperties) {
+      return new GCSBlobStorage(gcsStorage, configurationProperties);
+    }
+  }
+}