Add MD5 attribute

clientutil/parse.go

@@ -4,8 +4,10 @@
 package clientutil
 
 import (
+	"encoding/hex"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -63,3 +65,16 @@ func ParseLastModified(m http.Header, f string) (time.Time, error) {
 
 	return mod, nil
 }
+
+// ParseMD5 returns the bytes parsed from the hex-encoded MD5 string
+// It trims potential surrounding double quotes before decoding.
+// It returns nil if the MD5 string is not valid.
+func ParseMD5(md5Hex string) []byte {
+	// Trim surrounding double quotes if present.
+	trimmed := strings.Trim(md5Hex, "\"")
+	decoded, _ := hex.DecodeString(trimmed)
+	if len(decoded) != 16 {
+		return nil
+	}
+	return decoded
+}

clientutil/parse_test.go

@@ -4,6 +4,8 @@
 package clientutil
 
 import (
+	"bytes"
+	"fmt"
 	"net/http"
 	"testing"
 	"time"
@@ -108,3 +110,49 @@ func TestParseContentLength(t *testing.T) {
 		})
 	}
 }
+
+func TestParseMD5(t *testing.T) {
+	for _, tc := range []struct {
+		label string
+		in    string
+		out   []byte
+	}{
+		{
+			label: "valid md5",
+			in:    "b1946ac92492d2347c6235b4d2611184",
+			out:   []byte{0xb1, 0x94, 0x6a, 0xc9, 0x24, 0x92, 0xd2, 0x34, 0x7c, 0x62, 0x35, 0xb4, 0xd2, 0x61, 0x11, 0x84},
+		},
+		{
+			label: "invalid hex string",
+			in:    "not-a-hex-string",
+		},
+		{
+			label: "odd length hex string",
+			in:    "abc",
+		},
+		{
+			label: "empty string",
+			in:    "",
+			out:   []byte{},
+		},
+		{
+			label: "valid md5 with quotes",
+			in:    "\"b1946ac92492d2347c6235b4d2611184\"",
+			out:   []byte{0xb1, 0x94, 0x6a, 0xc9, 0x24, 0x92, 0xd2, 0x34, 0x7c, 0x62, 0x35, 0xb4, 0xd2, 0x61, 0x11, 0x84},
+		},
+		{
+			label: "invalid hex string with quotes",
+			in:    "\"not-a-hex-string\"",
+		},
+		{
+			label: "only quotes",
+			in:    "\"\"",
+			out:   []byte{},
+		},
+	} {
+		t.Run(tc.label, func(t *testing.T) {
+			out := ParseMD5(tc.in)
+			testutil.Assert(t, bytes.Equal(out, tc.out), fmt.Sprintf("output mismatch: %v != %v", out, tc.out))
+		})
+	}
+}

objstore.go

@@ -280,6 +280,9 @@ type ObjectAttributes struct {
 
 	// LastModified is the timestamp the object was last modified.
 	LastModified time.Time `json:"last_modified"`
+
+	// MD5 is the MD5 hash of the object, if available.
+	MD5 []byte `json:"md5,omitempty"`
 }
 
 type IterObjectAttributes struct {

providers/azure/azure.go

@@ -348,6 +348,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 	return objstore.ObjectAttributes{
 		Size:         *resp.ContentLength,
 		LastModified: *resp.LastModified,
+		MD5:          resp.ContentMD5,
 	}, nil
 }
 

providers/bos/bos.go

@@ -23,6 +23,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/clientutil"
 )
 
 // partSize 128MB.
@@ -313,6 +314,7 @@ func (b *Bucket) Attributes(_ context.Context, name string) (objstore.ObjectAttr
 	return objstore.ObjectAttributes{
 		Size:         objMeta.ContentLength,
 		LastModified: lastModified,
+		MD5:          clientutil.ParseMD5(objMeta.ContentMD5),
 	}, nil
 }
 

providers/cos/cos.go

@@ -185,6 +185,8 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 		return objstore.ObjectAttributes{}, err
 	}
 
+	// oss does not return md5 and etag is not necessarily md5 of the object content
+	// https://www.tencentcloud.com/document/product/436/7729
 	return objstore.ObjectAttributes{
 		Size:         size,
 		LastModified: mod,

providers/filesystem/filesystem.go

@@ -5,6 +5,7 @@ package filesystem
 
 import (
 	"context"
+	"crypto/md5"
 	"fmt"
 	"io"
 	"os"
@@ -171,9 +172,21 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 		return objstore.ObjectAttributes{}, errors.Wrapf(err, "stat %s", file)
 	}
 
+	f, err := os.Open(filepath.Clean(file))
+	if err != nil {
+		return objstore.ObjectAttributes{}, errors.Wrapf(err, "open file %s for md5 calculation", file)
+	}
+	defer f.Close()
+
+	h := md5.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return objstore.ObjectAttributes{}, errors.Wrapf(err, "copy file content for md5 calculation %s", file)
+	}
+
 	return objstore.ObjectAttributes{
 		Size:         stat.Size(),
 		LastModified: stat.ModTime(),
+		MD5:          h.Sum(nil),
 	}, nil
 }
 

providers/gcs/gcs.go

@@ -312,6 +312,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 	return objstore.ObjectAttributes{
 		Size:         attrs.Size,
 		LastModified: attrs.Updated,
+		MD5:          attrs.MD5,
 	}, nil
 }
 

providers/obs/obs.go

@@ -5,6 +5,7 @@ package obs
 
 import (
 	"context"
+	"encoding/base64"
 	"io"
 	"math"
 	"os"
@@ -373,9 +374,15 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 	if err != nil {
 		return objstore.ObjectAttributes{}, errors.Wrap(err, "failed to get object metadata")
 	}
+
+	// obs etag is base64 md5 of the object content for unencrypted objects
+	// https://support.huaweicloud.com/intl/en-us/sdk-go-devg-obs/obs_33_0519.html#section4
+	md5, _ := base64.StdEncoding.DecodeString(output.ETag)
+
 	return objstore.ObjectAttributes{
 		Size:         output.ContentLength,
 		LastModified: output.LastModified,
+		MD5:          md5,
 	}, nil
 }
 

providers/oci/oci.go

@@ -24,6 +24,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/clientutil"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -289,9 +290,16 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 	if err != nil {
 		return objstore.ObjectAttributes{}, err
 	}
+
+	var md5 []byte
+	if response.ContentMd5 != nil {
+		md5 = clientutil.ParseMD5(*response.ContentMd5)
+	}
+
 	return objstore.ObjectAttributes{
 		Size:         *response.ContentLength,
 		LastModified: response.LastModified.Time,
+		MD5:          md5,
 	}, nil
 }
 

providers/oss/oss.go

@@ -156,9 +156,17 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 		return objstore.ObjectAttributes{}, err
 	}
 
+	// etag is md5 of the object, if an object is created by calling the PutObject operation
+	// https://www.alibabacloud.com/help/en/oss/developer-reference/getobjectmeta
+	var md5 []byte
+	if etag := m.Get("ETag"); etag != "" {
+		md5 = clientutil.ParseMD5(etag)
+	}
+
 	return objstore.ObjectAttributes{
 		Size:         size,
 		LastModified: mod,
+		MD5:          md5,
 	}, nil
 }
 

providers/s3/s3.go

@@ -26,6 +26,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/clientutil"
 	"github.com/thanos-io/objstore/exthttp"
 )
 
@@ -601,6 +602,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt
 	return objstore.ObjectAttributes{
 		Size:         objInfo.Size,
 		LastModified: objInfo.LastModified,
+		MD5:          clientutil.ParseMD5(objInfo.ETag),
 	}, nil
 }
 

providers/swift/swift.go

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/clientutil"
 	"github.com/thanos-io/objstore/exthttp"
 	"gopkg.in/yaml.v2"
 )
@@ -310,9 +311,11 @@ func (c *Container) Attributes(_ context.Context, name string) (objstore.ObjectA
 	if err != nil {
 		return objstore.ObjectAttributes{}, errors.Wrap(err, "get object attributes")
 	}
+
 	return objstore.ObjectAttributes{
 		Size:         info.Bytes,
 		LastModified: info.LastModified,
+		MD5:          clientutil.ParseMD5(info.Hash),
 	}, nil
 }