Upload build artifacts using presigned URLs

src/main/java/io/papermc/fill/gradle/FillExtensionImpl.java

@@ -22,7 +22,6 @@
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.ProviderFactory;
 import org.jspecify.annotations.NullMarked;
-import java.time.Instant;
 
 @NullMarked
 public class FillExtensionImpl implements FillExtension {

src/main/java/io/papermc/fill/gradle/task/PublishToFillTask.java

@@ -24,28 +24,32 @@
 import io.papermc.fill.model.Checksums;
 import io.papermc.fill.model.Commit;
 import io.papermc.fill.model.Download;
-import io.papermc.fill.model.request.PublishRequest;
+import io.papermc.fill.model.request.v3.PublishRequest;
+import io.papermc.fill.model.request.v3.StageRequest;
 import io.papermc.fill.model.response.v3.BuildResponse;
+import io.papermc.fill.model.response.v3.StageResponse;
 import io.papermc.fill.model.response.v3.VersionResponse;
+import io.papermc.fill.model.response.v3.VersionsResponse;
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.function.Consumer;
 import javax.inject.Inject;
-import io.papermc.fill.model.response.v3.VersionsResponse;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.lib.Constants;
@@ -66,12 +70,13 @@
 @UntrackedTask(because = "PublishToFillTask should always run when requested")
 public abstract class PublishToFillTask extends DefaultTask implements AutoCloseable {
   public static final String NAME = "publishToFill";
+  private static final String APPLICATION_JAVA_ARCHIVE = "application/java-archive";
   private static final String USER_AGENT = "Fill (Gradle Plugin)";
   private final HttpClient httpClient = HttpClient.newBuilder()
     .build();
 
   public PublishToFillTask() {
     this.setGroup("fill");
     this.setDescription("Publish to Fill");
   }
 
@@ -102,7 +107,7 @@
     final String familyId = extension.getVersionFamily().get();
     final String versionId = extension.getVersion().get();
     final FillExtension.Build build = extension.getBuild();
-    final int buildId = build.getId().get();
+    final int buildNumber = build.getId().get();
     final String timeString = this.getExtension().get().getBuildTimestamp().getOrNull();
     final Instant time;
     if (timeString != null) {
@@ -117,93 +122,117 @@
 
     final List<Commit> commits = this.gatherCommits(git, extension);
 
+    if (!extension.getApiToken().isPresent()) {
+      throw new GradleException("API token is not present");
+    }
+    final String apiToken = extension.getApiToken().get();
     final UUID id = UUID.randomUUID();
-    final List<HttpRequest> requests = new ArrayList<>();
+    final Map<String, Download> downloads = new HashMap<>();
+    final List<PendingUpload> uploads = new ArrayList<>();
     try {
-      final Map<String, Download> downloads = new HashMap<>();
-
       for (final FillExtension.Download download : build.getDownloads()) {
         final String key = download.getName();
-        final String name = download.getNameResolver().get().name(project, familyId, versionId, buildId);
+        final String name = download.getNameResolver().get().name(project, familyId, versionId, buildNumber);
         final Path path = download.getFile().get().getAsFile().toPath();
-
         final byte[] content = Files.readAllBytes(path);
         final String sha256 = Hashing.sha256().hashBytes(content).toString();
-        final int size = (int) Files.size(path);
-        downloads.put(key, new Download(name, new Checksums(sha256), size));
-
-        final HttpRequest.Builder builder = HttpRequest.newBuilder();
-        builder.header("User-Agent", USER_AGENT);
-        builder.header("Content-Type", "multipart/form-data; boundary=boundary");
-        builder.uri(URI.create(extension.getApiUrl().get() + "/upload"));
-
-        final List<byte[]> requestParts = new ArrayList<>();
-        requestParts.add(("--boundary\r\nContent-Disposition: form-data; name=\"request\"\r\nContent-Type: application/json\r\n\r\n{\"id\":\"" + id + "\"}\r\n").getBytes(StandardCharsets.UTF_8));
-        requestParts.add(("--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"" + name + "\"\r\n\r\n").getBytes(StandardCharsets.UTF_8));
-        requestParts.add(content);
-        requestParts.add(("\r\n--boundary").getBytes(StandardCharsets.UTF_8));
-
-        builder.POST(HttpRequest.BodyPublishers.ofByteArrays(requestParts));
-
-        if (extension.getApiToken().isPresent()) {
-          builder.header("Authorization", extension.getApiToken().get());
-        } else {
-          throw new GradleException("API token is not present");
-        }
-
-        requests.add(builder.build());
+        final int size = content.length;
+        final Download requestDownload = new Download(name, new Checksums(sha256), size);
+        downloads.put(key, requestDownload);
+        uploads.add(new PendingUpload(requestDownload, content, APPLICATION_JAVA_ARCHIVE, contentMd5(content)));
       }
 
-      for (final HttpRequest request : requests) {
-        try {
-          final HttpResponse<String> response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString());
-          if (response.statusCode() != 200) {
-            throw new GradleException("Failed to post data to the API: " + response.statusCode() + ": " + response.body());
-          }
-        } catch (final Exception e) {
-          throw new GradleException("Failed to post data to the API", e);
-        }
+      for (final PendingUpload upload : uploads) {
+        final URI uploadUrl = this.requestUploadUrl(extension, apiToken, id, upload);
+        this.upload(uploadUrl, upload);
       }
 
       final PublishRequest request = new PublishRequest(
         id,
         project,
         familyId,
         versionId,
-        buildId,
+        buildNumber,
         time,
         build.getChannel().get(),
         commits.reversed(),
         downloads
       );
-
-      try {
-        final HttpRequest.Builder builder = HttpRequest.newBuilder()
-          .uri(URI.create(extension.getApiUrl().get() + "/publish"))
-          .header("Content-Type", "application/json")
-          .header("User-Agent", USER_AGENT);
-        builder.POST(HttpRequest.BodyPublishers.ofString(MapperHolder.MAPPER.writeValueAsString(request)));
-        if (extension.getApiToken().isPresent()) {
-          builder.headers("Authorization", extension.getApiToken().get());
-        } else {
-          throw new GradleException("Api token is not present.");
-        }
-
-        final HttpRequest post = builder.build();
-        final HttpResponse<String> response = this.httpClient.send(post, HttpResponse.BodyHandlers.ofString());
-        if (response.statusCode() != 201) {
-          throw new GradleException("Failed to post data to the API: " + response.statusCode() + ": " + response.body());
-        }
-      } catch (final Exception e) {
-        throw new GradleException("Failed to post data to the API: " + e.getMessage(), e);
-      }
+      this.publish(extension, apiToken, request);
     } catch (final JsonProcessingException e) {
-      throw new GradleException("Failed to serialize json", e);
+      throw new GradleException("Failed to serialize JSON", e);
     } catch (final IOException e) {
-      throw new GradleException("Failed to read file", e);
+      throw new GradleException("Failed to publish files", e);
+    } catch (final InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new GradleException("Publishing was interrupted", e);
     }
   }
 
+  private URI requestUploadUrl(
+    final FillExtension extension,
+    final String apiToken,
+    final UUID id,
+    final PendingUpload upload
+  ) throws IOException, InterruptedException {
+    final StageRequest request = new StageRequest(id, upload.download(), upload.contentType(), upload.contentMd5());
+    final HttpRequest httpRequest = HttpRequest.newBuilder()
+      .uri(URI.create(extension.getApiUrl().get() + "/v3/publishing/stage"))
+      .header("Authorization", apiToken)
+      .header("Content-Type", "application/json")
+      .header("User-Agent", USER_AGENT)
+      .POST(HttpRequest.BodyPublishers.ofString(MapperHolder.MAPPER.writeValueAsString(request)))
+      .build();
+    final HttpResponse<String> response = this.httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
+    if (response.statusCode() != 200) {
+      throw new IOException("Failed to create upload URL: " + response.statusCode() + ": " + response.body());
+    }
+    return MapperHolder.MAPPER.readValue(response.body(), StageResponse.class).url();
+  }
+
+  private void upload(final URI uploadUrl, final PendingUpload upload) throws IOException, InterruptedException {
+    final HttpRequest request = HttpRequest.newBuilder()
+      .uri(uploadUrl)
+      .header("Content-MD5", upload.contentMd5())
+      .header("Content-Type", upload.contentType())
+      .header("x-amz-meta-sha256", upload.download().checksums().sha256())
+      .PUT(HttpRequest.BodyPublishers.ofByteArray(upload.content()))
+      .build();
+    final HttpResponse<String> response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+    if (response.statusCode() / 100 != 2) {
+      throw new IOException("Failed to upload file: " + response.statusCode() + ": " + response.body());
+    }
+  }
+
+  private void publish(
+    final FillExtension extension,
+    final String apiToken,
+    final PublishRequest request
+  ) throws IOException, InterruptedException {
+    final HttpRequest httpRequest = HttpRequest.newBuilder()
+      .uri(URI.create(extension.getApiUrl().get() + "/v3/publishing/publish"))
+      .header("Authorization", apiToken)
+      .header("Content-Type", "application/json")
+      .header("User-Agent", USER_AGENT)
+      .POST(HttpRequest.BodyPublishers.ofString(MapperHolder.MAPPER.writeValueAsString(request)))
+      .build();
+    final HttpResponse<String> response = this.httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
+    if (response.statusCode() != 201) {
+      throw new IOException("Failed to publish build: " + response.statusCode() + ": " + response.body());
+    }
+  }
+
+  private static String contentMd5(final byte[] content) {
+    try {
+      return Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(content));
+    } catch (final NoSuchAlgorithmException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private record PendingUpload(Download download, byte[] content, String contentType, String contentMd5) {
+  }
+
   private List<Commit> gatherCommits(Git git, FillExtension extension) {
     final List<Commit> commits = new ArrayList<>();
     try (final RevWalk revWalk = new RevWalk(git.getRepository())) {

src/main/java/io/papermc/fill/model/AbstractDownload.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2024 PaperMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.papermc.fill.model;
+
+import org.jspecify.annotations.NullMarked;
+
+@NullMarked
+public interface AbstractDownload extends Named {
+  Checksums checksums();
+
+  int size();
+}

src/main/java/io/papermc/fill/model/Commit.java

@@ -18,11 +18,17 @@
 import java.time.Instant;
 import org.jspecify.annotations.NullMarked;
 
-// TODO: include author?
 @NullMarked
 public record Commit(
   String sha,
   Instant time,
   String message
 ) {
+  public static String getShortSha(final Commit commit) {
+    return commit.sha().substring(0, 7);
+  }
+
+  public String summary() {
+    return this.message.split("\\R")[0];
+  }
 }

src/main/java/io/papermc/fill/model/Download.java

@@ -15,12 +15,16 @@
  */
 package io.papermc.fill.model;
 
+import java.net.URI;
 import org.jspecify.annotations.NullMarked;
 
 @NullMarked
 public record Download(
   String name,
   Checksums checksums,
   int size
-) {
+) implements AbstractDownload {
+  public DownloadWithUrl withUrl(final URI url) {
+    return new DownloadWithUrl(this.name, this.checksums, this.size, url);
+  }
 }

src/main/java/io/papermc/fill/model/DownloadWithUrl.java

@@ -24,5 +24,5 @@ public record DownloadWithUrl(
   Checksums checksums,
   int size,
   URI url
-) {
+) implements AbstractDownload {
 }

src/main/java/io/papermc/fill/model/Java.java

@@ -15,23 +15,11 @@
  */
 package io.papermc.fill.model;
 
-import java.util.List;
 import org.jspecify.annotations.NullMarked;
 
 @NullMarked
 public record Java(
   JavaVersion version,
   JavaFlags flags
 ) {
-  @NullMarked
-  public record JavaVersion(
-    int minimum
-  ) {
-  }
-
-  @NullMarked
-  public record JavaFlags(
-    List<String> recommended
-  ) {
-  }
 }

src/main/java/io/papermc/fill/model/JavaFlags.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2024 PaperMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.papermc.fill.model;
+
+import java.util.List;
+import org.jspecify.annotations.NullMarked;
+
+@NullMarked
+public record JavaFlags(
+  List<String> recommended
+) {
+}

src/main/java/io/papermc/fill/model/request/UploadRequest.java → src/main/java/io/papermc/fill/model/JavaVersion.java

@@ -13,13 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.papermc.fill.model.request;
+package io.papermc.fill.model;
 
-import java.util.UUID;
 import org.jspecify.annotations.NullMarked;
 
 @NullMarked
-public record UploadRequest(
-  UUID id
+public record JavaVersion(
+  int minimum
 ) {
 }