perf: 使用并发优化上传效率

Author: SugarMGP

.golangci.yml

@@ -21,6 +21,8 @@ linters-settings:
     # 默认失败置信度
     confidence: 0.1
     rules:
+      - name: unchecked-type-assertion
+        disabled: true
       - name: var-naming            # 变量命名规则
         severity: warning
         disabled: true

internal/controllers/objectController/upload.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"mime/multipart"
 	"path/filepath"
+	"sync"
 
 	"github.com/gin-gonic/gin"
 	uuid "github.com/satori/go.uuid"
@@ -44,71 +45,79 @@ func BatchUploadFiles(c *gin.Context) {
 		return
 	}
 
-	results := make([]uploadFileRespElement, 0)
-	for _, fileHeader := range data.Files {
-		element := uploadFileRespElement{
-			Filename: fileHeader.Filename,
-		}
+	var mutex sync.Mutex
+	var wg sync.WaitGroup
+	results := make([]uploadFileRespElement, 0, len(data.Files))
+	for _, f := range data.Files {
+		wg.Add(1)
+		go func(fileHeader *multipart.FileHeader) {
+			defer wg.Done()
+
+			res := handleSingleUpload(fileHeader, &data, bucket, c.ClientIP())
+			mutex.Lock()
+			results = append(results, res)
+			mutex.Unlock()
+		}(f)
+	}
 
-		fileSize := fileHeader.Size
-		if fileSize > objectService.SizeLimit {
-			element.Error = apiException.FileSizeExceedError.Error()
-			results = append(results, element)
-			continue
-		}
+	wg.Wait()
+	response.JsonSuccessResp(c, gin.H{
+		"results": results,
+	})
+}
 
-		filename := fileHeader.Filename
-		ext := filepath.Ext(filename)             // 获取文件扩展名
-		name := filename[:len(filename)-len(ext)] // 获取去掉扩展名的文件名
+func handleSingleUpload(
+	fileHeader *multipart.FileHeader, data *batchUploadFileData, bucket oss.StorageProvider, ip string,
+) uploadFileRespElement {
+	element := uploadFileRespElement{
+		Filename: fileHeader.Filename,
+	}
 
-		// 若使用 UUID 作为文件名
-		if data.UseUUID {
-			name = uuid.NewV1().String()
-		}
+	if fileHeader.Size > objectService.SizeLimit {
+		element.Error = apiException.FileSizeExceedError.Error()
+		return element
+	}
 
-		file, err := fileHeader.Open()
-		if err != nil {
-			element.Error = apiException.UploadFileError.Error()
-			results = append(results, element)
-			continue
-		}
+	filename := fileHeader.Filename
+	ext := filepath.Ext(filename)             // 获取文件扩展名
+	name := filename[:len(filename)-len(ext)] // 获取去掉扩展名的文件名
 
-		// 转换到 WebP
-		var reader io.ReadSeeker = file
-		if data.ConvertWebP {
-			reader, err = objectService.ConvertToWebP(file)
-			ext = ".webp"
-			if errors.Is(err, image.ErrFormat) {
-				element.Error = apiException.FileNotImageError.Error()
-				results = append(results, element)
-				continue
-			}
-			if err != nil {
-				element.Error = apiException.ServerError.Error()
-				results = append(results, element)
-				continue
-			}
-		}
+	// 若使用 UUID 作为文件名
+	if data.UseUUID {
+		name = uuid.NewV1().String()
+	}
 
-		// 上传文件
-		objectKey := objectService.GenerateObjectKey(data.Location, name, ext)
-		err = bucket.SaveObject(reader, objectKey)
+	file, err := fileHeader.Open()
+	if err != nil {
+		element.Error = apiException.UploadFileError.Error()
+		return element
+	}
+	defer func() { _ = file.Close() }()
+
+	// 转换到 WebP
+	var reader io.ReadSeeker = file
+	if data.ConvertWebP {
+		reader, err = objectService.ConvertToWebP(file)
+		ext = ".webp"
+		if errors.Is(err, image.ErrFormat) {
+			element.Error = apiException.FileNotImageError.Error()
+			return element
+		}
 		if err != nil {
 			element.Error = apiException.ServerError.Error()
-			results = append(results, element)
-			continue
+			return element
 		}
+	}
 
-		element.ObjectKey = objectKey
-		results = append(results, element)
-
-		zap.L().Info("上传文件成功", zap.String("bucket", data.Bucket), zap.String("objectKey", objectKey), zap.String("ip", c.ClientIP()))
-
-		// 关闭文件
-		_ = file.Close()
+	// 上传文件
+	objectKey := objectService.GenerateObjectKey(data.Location, name, ext)
+	err = bucket.SaveObject(reader, objectKey)
+	if err != nil {
+		element.Error = apiException.ServerError.Error()
+		return element
 	}
 
-	response.JsonSuccessResp(c, gin.H{
-		"results": results,
-	})
+	element.ObjectKey = objectKey
+	zap.L().Info("上传文件成功", zap.String("bucket", data.Bucket), zap.String("objectKey", objectKey), zap.String("ip", ip))
+	return element
 }

internal/services/objectService/service.go

@@ -8,7 +8,9 @@ import (
 	_ "image/png"
 	"io"
 	"path"
+	"regexp"
 	"strings"
+	"sync"
 
 	"github.com/chai2010/webp"
 	"github.com/dustin/go-humanize"
@@ -21,6 +23,14 @@ import (
 // SizeLimit 上传大小限制
 var SizeLimit = humanize.MiByte * config.Config.GetInt64("oss.limit")
 
+var invalidCharRegex = regexp.MustCompile(`[:*?"<>|]`)
+
+var bufferPool = sync.Pool{
+	New: func() any {
+		return new(bytes.Buffer)
+	},
+}
+
 // GenerateObjectKey 通过路径和文件名生成 ObjectKey
 func GenerateObjectKey(location string, filename string, fileExt string) string {
 	return path.Join(CleanLocation(location), filename+fileExt)
@@ -29,12 +39,7 @@ func GenerateObjectKey(location string, filename string, fileExt string) string
 // CleanLocation 清理以避免非法路径
 func CleanLocation(location string) string {
 	isDir := strings.HasSuffix(location, "/")
-	loc := location
-	invalidChars := []string{":", "*", "?", "<", ">", "|", "\""}
-	for _, char := range invalidChars {
-		loc = strings.ReplaceAll(loc, char, "")
-	}
-
+	loc := invalidCharRegex.ReplaceAllString(location, "")
 	result := strings.TrimLeft(path.Clean(loc), "./\\")
 	if isDir {
 		result += "/"
@@ -49,8 +54,11 @@ func ConvertToWebP(reader io.Reader) (*bytes.Reader, error) {
 		return nil, err
 	}
 
-	var buf bytes.Buffer
-	err = webp.Encode(&buf, img, &webp.Options{Quality: 100})
+	buf, _ := bufferPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer bufferPool.Put(buf)
+
+	err = webp.Encode(buf, img, &webp.Options{Quality: 100})
 	if err != nil {
 		return nil, err
 	}

pkg/oss/local.go

@@ -119,7 +119,7 @@ func (p *LocalStorageProvider) GetFileList(prefix string) ([]FileListElement, er
 		return nil, err
 	}
 
-	list := make([]FileListElement, 0)
+	list := make([]FileListElement, 0, len(fileList))
 	for _, file := range fileList {
 		fileInfo, err := file.Info()
 		if err != nil {