all: oss: ListObjects should treat prefix as a dir

See:
- helm/chartmuseum#1082
- helm/chartmuseum#794

Signed-off-by: scnace <[email protected]>

Author: scbizu

alibaba.go

@@ -53,7 +53,6 @@ func NewAlibabaCloudOSSBackend(bucket string, prefix string, endpoint string, ss
 	}
 
 	client, err := oss.New(endpoint, accessKeyId, accessKeySecret)
-
 	if err != nil {
 		panic("Failed to create OSS client: " + err.Error())
 	}
@@ -76,7 +75,7 @@ func NewAlibabaCloudOSSBackend(bucket string, prefix string, endpoint string, ss
 func (b AlibabaCloudOSSBackend) ListObjects(prefix string) ([]Object, error) {
 	var objects []Object
 
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 	ossPrefix := oss.Prefix(prefix)
 	marker := oss.Marker("")
 	for {
@@ -113,7 +112,6 @@ func (b AlibabaCloudOSSBackend) GetObject(path string) (Object, error) {
 	var content []byte
 	key := pathutil.Join(b.Prefix, path)
 	body, err := b.Bucket.GetObject(key)
-
 	if err != nil {
 		return object, err
 	}

amazon.go

@@ -18,12 +18,13 @@ package storage
 
 import (
 	"bytes"
+	"crypto/tls"
 	"io/ioutil"
-	pathutil "path"
-	"strings"
 	"net/http"
-	"crypto/tls"
 	"os"
+	pathutil "path"
+	"strings"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -133,7 +134,7 @@ func NewAmazonS3BackendWithCredentials(bucket string, prefix string, region stri
 // ListObjects lists all objects in Amazon S3 bucket, at prefix
 func (b AmazonS3Backend) ListObjects(prefix string) ([]Object, error) {
 	var objects []Object
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 	s3Input := &s3.ListObjectsInput{
 		Bucket: aws.String(b.Bucket),
 		Prefix: aws.String(prefix),

baidu.go

@@ -52,7 +52,6 @@ func NewBaiDuBOSBackend(bucket string, prefix string, endpoint string) *BaiduBOS
 	}
 
 	client, err := bos.NewClient(accessKeyId, accessKeySecret, endpoint)
-
 	if err != nil {
 		panic("Failed to create BOS client: " + err.Error())
 	}
@@ -67,44 +66,44 @@ func NewBaiDuBOSBackend(bucket string, prefix string, endpoint string) *BaiduBOS
 
 // ListObjects lists all objects in Baidu Cloud BOS bucket, at prefix
 func (b BaiduBOSBackend) ListObjects(prefix string) ([]Object, error) {
-    var objects []Object
-
-    prefix = pathutil.Join(b.Prefix, prefix)
-    listObjectsArgs := &api.ListObjectsArgs{
-        Prefix:  prefix,
-        Marker:  "",
-        MaxKeys: 1000,
-    }
-    for {
-        lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs)
-        if err != nil {
-            return objects, err
-        }
-
-        for _, obj := range lor.Contents {
-            path := removePrefixFromObjectPath(prefix, obj.Key)
-            if objectPathIsInvalid(path) {
-                continue
-            }
-            lastModified, err := time.Parse(time.RFC3339, obj.LastModified)
-            if err != nil {
-                continue
-            }
-            object := Object{
-                Path:         path,
-                Content:      []byte{},
-                LastModified: lastModified,
-            }
-            objects = append(objects, object)
-        }
-        if !lor.IsTruncated {
-            break
-        }
-        listObjectsArgs.Prefix = lor.Prefix
-        listObjectsArgs.Marker = lor.NextMarker
-    }
-
-    return objects, nil
+	var objects []Object
+
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
+	listObjectsArgs := &api.ListObjectsArgs{
+		Prefix:  prefix,
+		Marker:  "",
+		MaxKeys: 1000,
+	}
+	for {
+		lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs)
+		if err != nil {
+			return objects, err
+		}
+
+		for _, obj := range lor.Contents {
+			path := removePrefixFromObjectPath(prefix, obj.Key)
+			if objectPathIsInvalid(path) {
+				continue
+			}
+			lastModified, err := time.Parse(time.RFC3339, obj.LastModified)
+			if err != nil {
+				continue
+			}
+			object := Object{
+				Path:         path,
+				Content:      []byte{},
+				LastModified: lastModified,
+			}
+			objects = append(objects, object)
+		}
+		if !lor.IsTruncated {
+			break
+		}
+		listObjectsArgs.Prefix = lor.Prefix
+		listObjectsArgs.Marker = lor.NextMarker
+	}
+
+	return objects, nil
 }
 
 // GetObject retrieves an object from Baidu Cloud BOS bucket, at prefix

etcd.go

@@ -2,24 +2,25 @@ package storage
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	pathutil "path"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
 const DefaultPrefix = "/chart_backend_bucket"
 
 var (
 	DefileDialTimeOut    = "5s"
 	TimeStampKey         = "timestamp"
-	ErrNotExistEndpoints = fmt.Errorf("endpoints cannot connect !")
-	ErrNotExist          = fmt.Errorf("not exist!")
+	ErrNotExistEndpoints = errors.New("endpoints cannot connect !")
+	ErrNotExist          = errors.New("not exist!")
 )
 
 type etcdOpts struct {
@@ -38,7 +39,7 @@ type etcdStorage struct {
 	mu     sync.RWMutex
 }
 
-//connection cut off
+// connection cut off
 func isServerErr(err error) bool {
 	if err != nil {
 		if err == context.Canceled {
@@ -52,11 +53,9 @@ func isServerErr(err error) bool {
 	return false
 }
 
-//prapare {basepath} dir
+// prapare {basepath} dir
 func (e *etcdStorage) probe() error {
-	var (
-		err error
-	)
+	var err error
 	ctx, cancel := context.WithCancel(e.ctx)
 	_, err = e.c.Put(ctx, e.base, "")
 	cancel()
@@ -100,13 +99,10 @@ func (e *etcdStorage) delTimeStamp(path string) error {
 	return err
 }
 
-//
 func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) {
-	var (
-		objs []Object
-	)
+	var objs []Object
 	ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout)
-	newpath := pathutil.Join(e.base, prefix)
+	newpath := pathutil.Join(e.base, normalizePath(prefix))
 	resps, err := e.c.Get(ctx, newpath, clientv3.WithPrefix())
 	cancel()
 	if err != nil {
@@ -118,7 +114,7 @@ func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) {
 			if objectPathIsInvalid(path) {
 				continue
 			}
-			//TODO need optimizate
+			// TODO need optimizate
 			if strings.HasSuffix(path, TimeStampKey) {
 				continue
 			}
@@ -134,13 +130,10 @@ func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) {
 		}
 	}
 	return objs, nil
-
 }
 
 func (e *etcdStorage) GetObject(path string) (Object, error) {
-	var (
-		modifytime time.Time
-	)
+	var modifytime time.Time
 	ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout)
 	newpath := pathutil.Join(e.base, path)
 	resps, err := e.c.Get(ctx, newpath)
@@ -164,9 +157,7 @@ func (e *etcdStorage) GetObject(path string) (Object, error) {
 }
 
 func (e *etcdStorage) PutObject(path string, content []byte) error {
-	var (
-		updatetime = time.Now()
-	)
+	updatetime := time.Now()
 	ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout)
 	newpath := pathutil.Join(e.base, path)
 	_, err := e.c.Put(ctx, newpath, string(content))
@@ -191,9 +182,7 @@ func (e *etcdStorage) DeleteObject(path string) error {
 }
 
 func parseConf(endpoints string, cafile, certfile, keyfile string, dialtime time.Duration) clientv3.Config {
-	var (
-		es []string
-	)
+	var es []string
 	if endpoints == "" {
 		panic(ErrNotExistEndpoints)
 	}
@@ -215,9 +204,7 @@ func parseConf(endpoints string, cafile, certfile, keyfile string, dialtime time
 }
 
 func NewEtcdCSBackend(endpoints string, cafile, certfile, keyfile string, prefix string) Backend {
-	var (
-		basepath string
-	)
+	var basepath string
 	DialTimeOut, _ := time.ParseDuration(DefileDialTimeOut)
 	cli, err := clientv3.New(parseConf(endpoints, cafile, certfile, keyfile, DialTimeOut))
 	if err != nil {
@@ -249,5 +236,4 @@ func NewEtcdCSBackend(endpoints string, cafile, certfile, keyfile string, prefix
 		panic(err)
 	}
 	return e
-
 }

google.go

@@ -52,7 +52,7 @@ func NewGoogleCSBackend(bucket string, prefix string) *GoogleCSBackend {
 // ListObjects lists all objects in Google Cloud Storage bucket, at prefix
 func (b GoogleCSBackend) ListObjects(prefix string) ([]Object, error) {
 	var objects []Object
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 	listQuery := &storage.Query{
 		Prefix: prefix,
 	}

microsoft.go

@@ -19,11 +19,10 @@ package storage
 import (
 	"errors"
 	"io/ioutil"
+	"os"
 	pathutil "path"
 	"time"
 
-	"os"
-
 	microsoft_storage "github.com/Azure/azure-sdk-for-go/storage"
 )
 
@@ -39,7 +38,6 @@ type MicrosoftBlobBackend struct {
 
 // NewMicrosoftBlobBackend creates a new instance of MicrosoftBlobBackend
 func NewMicrosoftBlobBackend(container string, prefix string) *MicrosoftBlobBackend {
-
 	// From the Azure portal, get your storage account name and key and set environment variables.
 	accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 	var serviceBaseURL, apiVersion string
@@ -78,7 +76,7 @@ func (b MicrosoftBlobBackend) ListObjects(prefix string) ([]Object, error) {
 	}
 
 	var params microsoft_storage.ListBlobsParameters
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 	params.Prefix = prefix
 
 	for {

openstack.go

@@ -224,7 +224,7 @@ func NewOpenstackOSBackendV1Auth(container string, prefix string, caCert string)
 func (b OpenstackOSBackend) ListObjects(prefix string) ([]Object, error) {
 	var objects []Object
 
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 	opts := &osObjects.ListOpts{
 		Full:   true,
 		Prefix: prefix,

oracle.go

@@ -43,7 +43,6 @@ type OracleCSBackend struct {
 
 // NewOracleCSBackend creates a new instance of OracleCSBackend
 func NewOracleCSBackend(bucket string, prefix string, region string, compartmentId string) *OracleCSBackend {
-
 	var config common.ConfigurationProvider
 	var err error
 
@@ -102,7 +101,6 @@ func NewOracleCSBackend(bucket string, prefix string, region string, compartment
 }
 
 func createBucket(ctx context.Context, c objectstorage.ObjectStorageClient, namespace string, bucket string, compartmentId string) (string, error) {
-
 	// Create the bucket
 	request := objectstorage.CreateBucketRequest{
 		NamespaceName: &namespace,
@@ -115,7 +113,6 @@ func createBucket(ctx context.Context, c objectstorage.ObjectStorageClient, name
 	_, err := c.CreateBucket(ctx, request)
 
 	return bucket, err
-
 }
 
 func getNamespace(ctx context.Context, c objectstorage.ObjectStorageClient) (string, error) {
@@ -131,7 +128,7 @@ func getNamespace(ctx context.Context, c objectstorage.ObjectStorageClient) (str
 // ListObjects lists all objects in OCI Object Storage bucket, at prefix
 func (b OracleCSBackend) ListObjects(prefix string) ([]Object, error) {
 	var objects []Object
-	prefix = pathutil.Join(b.Prefix, prefix)
+	prefix = pathutil.Join(b.Prefix, normalizePath(prefix))
 
 	request := objectstorage.ListObjectsRequest{
 		NamespaceName: &b.Namespace,
@@ -181,14 +178,12 @@ func (b OracleCSBackend) GetObject(path string) (Object, error) {
 	}
 
 	rc, err := b.Client.GetObject(b.Context, request)
-
 	if err != nil {
 		return object, err
 	}
 
 	object.LastModified = rc.LastModified.Time
 	content, err := ioutil.ReadAll(rc.Content)
-
 	if err != nil {
 		return object, err
 	}
@@ -198,7 +193,6 @@ func (b OracleCSBackend) GetObject(path string) (Object, error) {
 
 // PutObject uploads an object to OCI Object Storage bucket, at prefix
 func (b OracleCSBackend) PutObject(path string, content []byte) error {
-
 	objectname := pathutil.Join(b.Prefix, path)
 	metadata := make(map[string]string)
 	contentLen := int64(binary.Size(content))
@@ -219,7 +213,6 @@ func (b OracleCSBackend) PutObject(path string, content []byte) error {
 
 // DeleteObject removes an object from OCI Object Storage bucket, at prefix
 func (b OracleCSBackend) DeleteObject(path string) error {
-
 	objectname := pathutil.Join(b.Prefix, path)
 
 	request := objectstorage.DeleteObjectRequest{

storage.go

@@ -97,6 +97,13 @@ func cleanPrefix(prefix string) string {
 	return strings.Trim(prefix, "/")
 }
 
+func normalizePath(path string) string {
+	if path != "" && !strings.HasSuffix(path, "/") {
+		path += "/"
+	}
+	return path
+}
+
 func removePrefixFromObjectPath(prefix string, path string) string {
 	if prefix == "" {
 		return path