支持上传到OneDrive任意文件夹

Author: gaowanliang

.gitignore

@@ -7,4 +7,5 @@ dist
 info
 *.json
 *.exe
-od
+od
+TestUploadFolder

Readme.md

@@ -4,7 +4,7 @@
 
 
 - 支持 国际版, 个人版(家庭版), ~~中国版(世纪互联)~~.
-- ~~支持上传文件和文件夹到指定目录,并~~ 保持上传前的目录结构.
+- 支持上传文件和文件夹到指定目录,并保持上传前的目录结构.
 - 支持命令参数使用, 方便外部程序调用.
 - 支持自定义上传分块大小.
 - 支持多线程上传(多文件同时上传).
@@ -53,7 +53,6 @@ Usage of OneDriveUploader:
         // 上传单个文件时,在网盘中重命名
         Rename file on upload to remote.
 
-  //此参数未设计，暂不可用
   -r string
         // 上传到网盘中的某个目录, 默认: 根目录
         Upload to reomte path.
@@ -112,30 +111,27 @@ Usage of OneDriveUploader:
 OneDriveUploader -c xxx.json -f "mm00.jpg"
 
 # 将同目录下的 mm00.jpg 文件上传到 OneDrive 网盘根目录,并改名为 mm01.jpg(暂不可用)
-OneDriveUploader  -c xxx.json -s "mm00.jpg" -n "mm01.jpg"
+OneDriveUploader  -c xxx.json -f "mm00.jpg" -n "mm01.jpg"
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录
-OneDriveUploader -c xxx.json -s "Download" 
+OneDriveUploader -c xxx.json -f "Download" 
 
-# 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中(暂不可用)
-OneDriveUploader -c xxx.json -s "Download" -r "Test"
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中
+OneDriveUploader -c xxx.json -f "Download" -r "Test"
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程
-OneDriveUploader -c xxx.json -t 10 -s "Download" 
+OneDriveUploader -c xxx.json -t 10 -f "Download" 
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程，同时使用 Telegram Bot 实时监控上传进度
-OneDriveUploader -c xxx.json -t 10 -s "Download" -tgbot "123456:xxxxxxxx" -uid 123456789
+OneDriveUploader -c xxx.json -t 10 -f "Download" -tgbot "123456:xxxxxxxx" -uid 123456789
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 15 线程, 并设置分块大小为 20M
-OneDriveUploader -c xxx.json -t 15 -b 20 -s "Download" 
+OneDriveUploader -c xxx.json -t 15 -b 20 -f "Download" 
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中, 使用配置文件中的线程参数和分块大小参数(暂不可用)
-OneDriveUploader -f -c "/urs/local/auth.json" -s "Download" -r "Test"
+OneDriveUploader -c "/urs/local/auth.json" -f "Download" -r "Test" -force
 
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中, 使用配置文件中的线程参数和分块大小参数，并跳过上传网盘中已存在的同名文件(暂不可用)
-OneDriveUploader -f -c "/urs/local/auth.json" -skip -s "Download" -r "Test"
+OneDriveUploader -c "/urs/local/auth.json" -skip -f "Download" -r "Test"
 ```
 
-## 注意
-- ~~多次尝试后, 无失败的上传文件. 退出码为 0 .~~
-- ~~最终还有失败的上传文件会详细列出上传失败项. 退出码为 1.~~

api/restore/upload/onedriveRecoverableRestore.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"main/fileutil"
 	"net/http"
+	"path/filepath"
 	"runtime/debug"
 	"strconv"
 	"time"
@@ -16,9 +17,9 @@ const (
 	uploadURLKey      = "uploadUrl"
 )
 
-func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, conflictOption string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) []map[string]interface{} {
+func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, conflictOption string, targetFolder string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) []map[string]interface{} {
 	//1. Get recoverable upload session for the current file path 获取当前文件路径的可压缩上载会话
-	uploadSessionData, err := rs.getUploadSession(userID, bearerToken, conflictOption, filePath)
+	uploadSessionData, err := rs.getUploadSession(userID, bearerToken, conflictOption, targetFolder, filePath)
 	if err != nil {
 		log.Panicf("Failed to Restore :%v", err)
 	}
@@ -31,6 +32,12 @@ func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, c
 	if err != nil {
 		log.Panicf("Failed to Restore :%v", err)
 	}
+	_size, err := fileutil.GetFileSize(filePath)
+	if err != nil {
+		log.Panicf("Failed to Restore :%v", err)
+	}
+	//log.Panicln(fileutil.Byte2Readable(size))
+	size := byte2Readable(float64(_size))
 
 	//4. Loop over the file start offset list to read files in chunk and upload in onedrive 在文件开始偏移量列表上循环以读取块中的文件并在onedrive中上载
 	var uploadResp []map[string]interface{}
@@ -39,6 +46,7 @@ func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, c
 	timeUnix := time.Now().UnixNano()
 	var buffer = make([]byte, fileutil.GetDefaultChunkSize())
 	startTime := time.Now().Unix()
+
 	for i, sOffset := range startOffsetLst {
 		if i == lastChunkIndex {
 			lastChunkSize, err := fileutil.GetLatsChunkSizeInBytes(filePath)
@@ -55,9 +63,9 @@ func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, c
 			log.Panicf("Failed to Restore :%v", err)
 		}
 		if i != 0 {
-			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s` *『%d/%d』*  \n速度:`%s/s` \n已耗时: `%d s`", username, filePath, i, len(startOffsetLst), byte2Readable(float64(fileutil.GetDefaultChunkSize())/float64(time.Now().UnixNano()-timeUnix)*float64(1000000000)), time.Now().Unix()-startTime))
+			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s`\n 大小:`%s` 已上传大小: `%s` 进度: *『%d/%d』*  \n速度:`%s/s` \n已耗时: `%d s`", username, filePath, size, byte2Readable(float64(fileutil.GetDefaultChunkSize())*float64(i)), i, len(startOffsetLst), byte2Readable(float64(fileutil.GetDefaultChunkSize())/float64(time.Now().UnixNano()-timeUnix)*float64(1000000000)), time.Now().Unix()-startTime))
 		} else {
-			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s` *『%d/%d』*  \n速度:`----` \n已耗时: `%d s`", username, filePath, i, len(startOffsetLst), time.Now().Unix()-startTime))
+			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s`\n 大小:`%s` 已上传大小: `%s` 进度: *『%d/%d』*  \n速度:`----` \n已耗时: `%d s`", username, filePath, size, "0", i, len(startOffsetLst), time.Now().Unix()-startTime))
 		}
 
 		timeUnix = time.Now().UnixNano()
@@ -92,8 +100,9 @@ func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, c
 }
 
 //Returns the restore session url for part file upload
-func (rs *RestoreService) getUploadSession(userID string, bearerToken string, conflictOption string, filePath string) (map[string]interface{}, error) {
-	uploadSessionPath := fmt.Sprintf(uploadSessionPath, userID, filePath)
+func (rs *RestoreService) getUploadSession(userID string, bearerToken string, conflictOption string, targetFolder string, filePath string) (map[string]interface{}, error) {
+	targetPath := filepath.Join(targetFolder, filePath)
+	uploadSessionPath := fmt.Sprintf(uploadSessionPath, userID, targetPath)
 	uploadSessionData := make(map[string]interface{})
 	//Get the body for resemble upload session call.
 	body, err := getRessumableSessionBody(filePath, conflictOption)

api/restore/upload/onedriveRestore.go

@@ -9,6 +9,7 @@ import (
 	httpLocal "main/graph/net/http"
 	"net/http"
 	"net/url"
+	"path/filepath"
 )
 
 const (
@@ -35,16 +36,17 @@ type RestoreService struct {
 //@bearerToken will be extracted as sent from the restore input xml
 //@filePath will be extracted from the file hierarchy the needs to be restored
 //@fileInfo it is the file info struct that contains the actual file reference and the size_type
-func (rs *RestoreService) SimpleUploadToOriginalLoc(userId string, bearerToken string, conflictOption string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) interface{} {
+func (rs *RestoreService) SimpleUploadToOriginalLoc(userId string, bearerToken string, conflictOption string, targetFolder string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) interface{} {
 	if fileInfo.SizeType == fileutil.SizeTypeLarge {
 		//For Large file type use resemble onedrive upload API
 		//log.Printf("Processing Large File: %s", filePath)
 		sendMsg(fmt.Sprintf("文件: `%s` 开始上传至OneDrive\n账户:`%s`\n文件超过4MB，进入大文件通道", filePath, username))
-		return rs.recoverableUpload(userId, bearerToken, conflictOption, filePath, fileInfo, sendMsg, locText, username)
+		return rs.recoverableUpload(userId, bearerToken, conflictOption, targetFolder, filePath, fileInfo, sendMsg, locText, username)
 	} else {
 		//log.Printf("Processing Small File: %s", filePath)
 		sendMsg(fmt.Sprintf("文件: `%s` 开始上传至OneDrive\n账户:`%s`\n文件小于4MB，进入小文件通道，上传中", filePath, username))
-		uploadPath := fmt.Sprintf(simpleUploadPath, userId, filePath)
+		targetPath := filepath.Join(targetFolder, filePath)
+		uploadPath := fmt.Sprintf(simpleUploadPath, userId, targetPath)
 		req, err := rs.NewRequest("PUT", uploadPath, getSimpleUploadHeader(bearerToken), fileInfo.FileData)
 		if err != nil {
 			log.Panicf("Failed to Restore :%v", err)
@@ -56,7 +58,16 @@ func (rs *RestoreService) SimpleUploadToOriginalLoc(userId string, bearerToken s
 		req.URL.RawQuery = q.Encode()
 
 		//Execute the request
-		resp, err := rs.Do(req)
+
+		var resp *http.Response
+		for errCount := 1; errCount < 10; errCount++ {
+			resp, err = rs.Do(req)
+			if err != nil {
+				sendMsg(fmt.Sprintf("向OneDrive账户 `%s` 上传 `%s` 时出现连接问题，正在重试，当前为第%d次重试", username, filePath, errCount))
+			} else {
+				break
+			}
+		}
 		if err != nil {
 			log.Panicf("Failed to Restore :%v", err)
 		}
@@ -81,10 +92,10 @@ func (rs *RestoreService) SimpleUploadToOriginalLoc(userId string, bearerToken s
 //@userId will be extracted as sent from the restore input xml
 //@filePath will be extracted from the file hierarchy the needs to be restored
 //@fileInfo it is the file info struct that contains the actual file reference and the size_type
-func (rs *RestoreService) SimpleUploadToAlternateLoc(altUserId string, bearerToken string, conflictOption string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) interface{} {
+func (rs *RestoreService) SimpleUploadToAlternateLoc(altUserId string, bearerToken string, targetFolder string, conflictOption string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) interface{} {
 	if fileInfo.SizeType == fileutil.SizeTypeLarge {
 		//For Large file type use resemble onedrive upload API
-		return rs.recoverableUpload(altUserId, bearerToken, conflictOption, filePath, fileInfo, sendMsg, locText, username)
+		return rs.recoverableUpload(altUserId, bearerToken, conflictOption, targetFolder, filePath, fileInfo, sendMsg, locText, username)
 	} else {
 
 		uploadPath := fmt.Sprintf(simpleUploadPath, altUserId, filePath)

fileutil/file.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 )
 
@@ -63,7 +64,7 @@ func GetAllUploadItemsFrmSource(sourcePath string) (map[string]FileInfo, error)
 }
 
 //GetFilePartInBytes can returns the file in parts based on the provided offset
-func GetFilePartInBytes(buffer *[]byte,filePath string, startingOffset int64) error {
+func GetFilePartInBytes(buffer *[]byte, filePath string, startingOffset int64) error {
 	file, err := os.Open(filePath)
 	defer file.Close()
 
@@ -87,7 +88,7 @@ func GetFilePartInBytes(buffer *[]byte,filePath string, startingOffset int64) er
 			return fmt.Errorf("readAt: %v", err)
 		}
 	}
-	return  nil
+	return nil
 }
 
 //Returns the start offset chunk list based on the file size
@@ -171,3 +172,27 @@ func ReadFile(file *os.File) ([]byte, error) {
 	}
 	return buffer, nil
 }
+
+func Byte2Readable(bytes float64) string {
+	const kb float64 = 1024
+	const mb = kb * 1024
+	const gb = mb * 1024
+	var readable float64
+	var unit string
+	_bytes := bytes
+
+	if _bytes >= gb {
+		// xx GB
+		readable = _bytes / gb
+		unit = "GB"
+	} else if _bytes < gb && _bytes >= mb {
+		// xx MB
+		readable = _bytes / mb
+		unit = "MB"
+	} else {
+		// xx KB
+		readable = _bytes / kb
+		unit = "KB"
+	}
+	return strconv.FormatFloat(readable, 'f', 2, 64) + " " + unit
+}

main.go

@@ -25,7 +25,7 @@ func ApplyForNewPass(url string, ms int) string {
 	return httpLocal.NewPassCheck(url, ms)
 }
 
-func Upload(infoPath string, filePath string, threads int, sendMsg func() (func(text string), string, string), locText func(text string) string) {
+func Upload(infoPath string, filePath string, targetFolder string, threads int, sendMsg func() (func(text string), string, string), locText func(text string) string) {
 
 	programPath, err := filepath.Abs(filepath.Dir(infoPath))
 	if err != nil {
@@ -59,7 +59,7 @@ func Upload(infoPath string, filePath string, threads int, sendMsg func() (func(
 	} else {
 		restore(restoreSrvc, fileInfoToUpload, threads)
 	}*/
-	restore(restoreSrvc, fileInfoToUpload, threads, sendMsg, locText, infoPath)
+	restore(restoreSrvc, fileInfoToUpload, targetFolder, threads, sendMsg, locText, infoPath)
 	err = os.Chdir(oldDir)
 	if err != nil {
 		log.Panic(err)
@@ -70,7 +70,7 @@ func changeBlockSize(MB int) {
 }
 
 //Restore to original location
-func restore(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileutil.FileInfo, threads int, sendMsg func() (func(text string), string, string), locText func(text string) string, infoPath string) {
+func restore(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileutil.FileInfo, targetFolder string, threads int, sendMsg func() (func(text string), string, string), locText func(text string) string, infoPath string) {
 	var wg sync.WaitGroup
 	pool := make(chan struct{}, threads)
 	for filePath, fileInfo := range filesToRestore {
@@ -96,7 +96,7 @@ func restore(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileu
 			temp(tip)
 			userID, bearerToken := httpLocal.GetMyIDAndBearer(infoPath)
 			username := strings.ReplaceAll(filepath.Base(infoPath), ".json", "")
-			restoreSrvc.SimpleUploadToOriginalLoc(userID, bearerToken, "rename", filePath, fileInfo, temp, locText, username)
+			restoreSrvc.SimpleUploadToOriginalLoc(userID, bearerToken, "rename", targetFolder, filePath, fileInfo, temp, locText, username)
 
 			//printResp(resp)
 			defer temp("close")
@@ -119,7 +119,7 @@ func printResp(resp interface{}) {
 }
 
 //Restore to Alternate location 还原到备用位置
-func restoreToAltLoc(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileutil.FileInfo, sendMsg func() func(text string), locText func(text string) string, infoPath string) {
+func restoreToAltLoc(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileutil.FileInfo, targetFolder string, sendMsg func() func(text string), locText func(text string) string, infoPath string) {
 	rootFolder := fileutil.GetAlternateRootFolder()
 	var wg sync.WaitGroup
 	pool := make(chan struct{}, 10)
@@ -137,7 +137,7 @@ func restoreToAltLoc(restoreSrvc *upload.RestoreService, filesToRestore map[stri
 			us := ""
 			userID, bearerToken := httpLocal.GetMyIDAndBearer(infoPath)
 			//username := strings.ReplaceAll(filepath.Base(infoPath), ".json", "")
-			restoreSrvc.SimpleUploadToAlternateLoc(userID, bearerToken, "rename", rootFilePath, fileItem, temp, locText, us)
+			restoreSrvc.SimpleUploadToAlternateLoc(userID, bearerToken, "rename", targetFolder, rootFilePath, fileItem, temp, locText, us)
 
 		}()
 		wg.Wait()
@@ -197,6 +197,18 @@ func botSend(botKey string, iuserID string, initText string) func(string) {
 
 	}
 }
+
+func DirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	return size, err
+}
+
 func main() {
 	var codeURL string
 	var configFile string
@@ -206,6 +218,7 @@ func main() {
 	var iuserID string
 	var ms int
 	var block int
+	var targetFolder string
 	// StringVar用指定的名称、控制台参数项目、默认值、使用信息注册一个string类型flag，并将flag的值保存到p指向的变量
 	flag.StringVar(&codeURL, "a", "", "通过登录 https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=ad5e65fd-856d-4356-aefc-537a9700c137&response_type=code&redirect_uri=http://localhost/onedrive-login&response_mode=query&scope=offline_access%20User.Read%20Files.ReadWrite.All 后跳转的网址(输入网址时需要使用双引号包裹网址)")
 	flag.IntVar(&ms, "v", 0, "选择版本，其中0为国际版，1为个人版(家庭版)，默认为0")
@@ -215,6 +228,7 @@ func main() {
 	flag.IntVar(&block, "b", 10, "自定义上传分块大小, 可以提高网络吞吐量, 受限于磁盘性能和网络速度，默认为 10 (单位MB)")
 	flag.StringVar(&botKey, "tgbot", "", "使用Telegram机器人实时监控上传，此处需填写机器人的access token，形如123456789:xxxxxxxxx，输入时需使用双引号包裹")
 	flag.StringVar(&iuserID, "uid", "", "使用Telegram机器人实时监控上传，此处需填写接收人的userID，形如123456789")
+	flag.StringVar(&targetFolder, "r", "", "设置上传的目录")
 
 	// 从arguments中解析注册的flag。必须在所有flag都注册好而未访问其值时执行。未注册却使用flag -help时，会返回ErrHelp。
 
@@ -224,12 +238,18 @@ func main() {
 		startTime := time.Now().Unix()
 		writer := uilive.New()
 		writer.Start()
+		size, err := DirSize(folder)
+		if err != nil {
+			log.Panic(err)
+		}
+		_, _ = fmt.Fprintf(writer, "`%s` 开始上传，大小: `%s`", folder, fileutil.Byte2Readable(float64(size)))
+
 		var sendMsg func(string)
 		if botKey != "" && iuserID != "" {
-			sendMsg = botSend(botKey, iuserID, fmt.Sprintf("`%s` 开始上传", folder))
+			sendMsg = botSend(botKey, iuserID, fmt.Sprintf("`%s` 开始上传，大小: `%s`", folder, fileutil.Byte2Readable(float64(size))))
 		}
 
-		Upload(strings.ReplaceAll(configFile, "\\", "/"), strings.ReplaceAll(folder, "\\", "/"), thread, func() (func(text string), string, string) {
+		Upload(strings.ReplaceAll(configFile, "\\", "/"), strings.ReplaceAll(folder, "\\", "/"), targetFolder, thread, func() (func(text string), string, string) {
 			if botKey != "" && iuserID != "" {
 				return func(text string) {
 					_, _ = fmt.Fprintf(writer, "%s\n", text)
@@ -243,9 +263,11 @@ func main() {
 		}, func(text string) string {
 			return oldFunc(text)
 		})
-		_, _ = fmt.Fprintf(writer, "%s上传完成，耗时 %d 秒\n", folder, time.Now().Unix()-startTime)
+		cost := time.Now().Unix() - startTime
+		speed := fileutil.Byte2Readable(float64(size) / float64(cost))
+		_, _ = fmt.Fprintf(writer, "%s上传完成，耗时 %d 秒，平均上传速度 %s/s\n", folder, cost, speed)
 		if botKey != "" && iuserID != "" {
-			sendMsg(fmt.Sprintf("`%s`上传完成，耗时 %d 秒\n", folder, time.Now().Unix()-startTime))
+			sendMsg(fmt.Sprintf("`%s`上传完成，耗时 %d 秒，平均上传速度 `%s/s`", folder, cost, speed))
 		}
 	} else {
 		if codeURL == "" {
@@ -257,6 +279,7 @@ func main() {
 
 	// 打印
 	//fmt.Printf("username=%v password=%v host=%v port=%v", username, password, host, port)
+
 }
 
 /*