From b6ae4a9804cb9a917d3acc6526f637ec216cf554 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 24 Apr 2026 17:53:52 +0200 Subject: [PATCH 1/2] fix(upload): opt out of flexible checksum headers for SDA inbox compatibility After #682 moved uploads to feature/s3/transfermanager, single-part PUTs started carrying x-amz-sdk-checksum-algorithm / x-amz-checksum-* / x-amz-trailer headers even though the s3 client is configured with RequestChecksumCalculationWhenRequired. transfermanager.New resolves an unset ChecksumAlgorithm to CRC32, after which its own value wins over the s3 client setting on PutObjectInput. Reset ChecksumAlgorithm to "" via a per-call optFn on UploadObject; the empty value also propagates to CreateMultipartUpload and UploadPart, so the opt-out covers both single- and multipart paths. New test wraps the gofakes3 mock to capture real PUT headers and asserts none of the flexible-checksum headers are sent. --- upload/upload.go | 3 +++ upload/upload_test.go | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/upload/upload.go b/upload/upload.go index 395701c8..19d0256d 100644 --- a/upload/upload.go +++ b/upload/upload.go @@ -229,6 +229,9 @@ func uploadFiles(files, outFiles []string, targetDir string, config *helpers.Con Bucket: aws.String(config.AccessKey), Key: aws.String(path.Join(targetDir, outFiles[k])), ContentEncoding: aws.String(config.Encoding), + }, func(o *transfermanager.Options) { + // Preserve RequestChecksumCalculationWhenRequired for S3-compatible inboxes. + o.ChecksumAlgorithm = "" }) // Print the progress bar. Second check is to filter out some junk from the output if result != nil && result.VersionID != nil { diff --git a/upload/upload_test.go b/upload/upload_test.go index 28ccd714..7b10a2af 100644 --- a/upload/upload_test.go +++ b/upload/upload_test.go @@ -6,10 +6,12 @@ import ( "crypto/rsa" "fmt" "io" + "net/http" "net/http/httptest" "os" "path/filepath" "runtime" + "sync" "testing" "time" @@ -36,6 +38,8 @@ type UploadTestSuite struct { publicKeyFilePath string s3MockHTTPServer *httptest.Server s3Client *s3.Client + objectPutHeaders http.Header + objectPutHeadersMu sync.Mutex } var configFileFormat = ` @@ -70,12 +74,24 @@ func (s *UploadTestSuite) SetupTest() { uploadCmd.Flag("access-token").Value.Set("") os.Args = []string{"", "upload"} + s.objectPutHeadersMu.Lock() + s.objectPutHeaders = nil + s.objectPutHeadersMu.Unlock() + s.accessToken = s.generateDummyToken() s.tempDir = s.T().TempDir() backend := s3mem.New() faker := gofakes3.New(backend) - s.s3MockHTTPServer = httptest.NewServer(faker.Server()) + fakeS3Handler := faker.Server() + s.s3MockHTTPServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && r.URL.Path != "/dummy" { + s.objectPutHeadersMu.Lock() + s.objectPutHeaders = r.Header.Clone() + s.objectPutHeadersMu.Unlock() + } + fakeS3Handler.ServeHTTP(w, r) + })) awsConfig, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("dummy", "dummy", s.accessToken)), @@ -275,6 +291,23 @@ func (s *UploadTestSuite) TestUploadTargetDir() { } assert.Equal(s.T(), aws.ToString(result.Contents[0].Key), fmt.Sprintf("%s/%s", filepath.ToSlash(targetPath), filepath.Base(s.uploadTestFilePath))) } + +func (s *UploadTestSuite) TestUploadSinglePartDoesNotSendFlexibleChecksumHeaders() { + os.Args = []string{"", "upload", s.uploadTestFilePath} + uploadCmd.Flag("force-unencrypted").Value.Set("true") + assert.NoError(s.T(), uploadCmd.Execute()) + + s.objectPutHeadersMu.Lock() + headers := s.objectPutHeaders.Clone() + s.objectPutHeadersMu.Unlock() + + if assert.NotNil(s.T(), headers, "expected object PUT request headers to be captured") { + assert.Empty(s.T(), headers.Values("X-Amz-Sdk-Checksum-Algorithm")) + assert.Empty(s.T(), headers.Values("X-Amz-Checksum-Crc32")) + assert.Empty(s.T(), headers.Values("X-Amz-Trailer")) + } +} + func (s *UploadTestSuite) TestUploadWithEncryption() { rescuedStdout := os.Stdout stdoutReader, stdoutWriter, _ := os.Pipe() From f8dae2ceaea1522ac56cd4609c2bc2e574eaf61d Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 28 May 2026 10:36:44 +0200 Subject: [PATCH 2/2] fix(upload): use explicit types.ChecksumAlgorithm conversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback from @nanjiangshu — assign the empty value through the named type rather than relying on untyped-string-to-named- type assignability. Functionally identical, just clearer at the call site. --- upload/upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/upload/upload.go b/upload/upload.go index 19d0256d..4413b41c 100644 --- a/upload/upload.go +++ b/upload/upload.go @@ -18,6 +18,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager/types" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/neicnordic/crypt4gh/keys" "github.com/spf13/cobra" @@ -231,7 +232,7 @@ func uploadFiles(files, outFiles []string, targetDir string, config *helpers.Con ContentEncoding: aws.String(config.Encoding), }, func(o *transfermanager.Options) { // Preserve RequestChecksumCalculationWhenRequired for S3-compatible inboxes. - o.ChecksumAlgorithm = "" + o.ChecksumAlgorithm = types.ChecksumAlgorithm("") }) // Print the progress bar. Second check is to filter out some junk from the output if result != nil && result.VersionID != nil {