Fix Google drive file upload problem

Author: gaowanliang

Readme-zh-CN.md

@@ -2,7 +2,7 @@
 
 萌咖大佬写了一个 [非常好的版本](https://github.com/MoeClub/OneList/tree/master/OneDriveUploader) ，可惜并没有开源，而且已经好久都没有更新了。这个项目作为从 [DownloadBot](https://github.com/gaowanliang/DownloadBot) 中独立出来的一个简易上传工具，旨在用更轻量化的方式让在各种平台都能快速的向各个网络硬盘上传数据。
 
-- 支持 OneDrive 国际版, 个人版(家庭版)，世纪互联，Google Drive（测试版，下面的命令并未完全支持）.
+- 支持 OneDrive 国际版, 个人版(家庭版)，世纪互联，Google Drive.
 - 支持上传文件和文件夹到指定目录,并保持上传前的目录结构.
 - 支持命令参数使用, 方便外部程序调用.
 - 支持自定义上传分块大小.
@@ -24,15 +24,15 @@ LightUploader -a "url" -l zh-CN
 # OneDrive 个人版(家庭版)
 LightUploader -a "url" -v 1
 # OneDrive 中国版(世纪互联)，并使用中文语言包
-LightUploader -a "url" -v 2
+LightUploader -a "url" -v 2 -l zh-CN
 
 # 在浏览器地址栏中获取以 http://loaclhost 开头的整个url内容
 # 将获取的完整url内容替换命令中的 url 三个字母
 # 每次产生的 url 只能用一次, 重试请重新获取 url
 # 此操作将会自动初始化的配置文件
 
 # Google Drive
-LightUploader -a "url" -v 3
+LightUploader -v 3
 ```
 
 ## 使用

Readme.md

@@ -5,7 +5,7 @@ MoeClub wrote a [very good version](https://github.com/MoeClub/OneList/tree/mast
 
 ## Features
 
-- Supports OneDrive Business, Personal (Home) versions, 21vianet (CN) version, Google Drive (Beta).
+- Supports OneDrive Business, Personal (Home) versions, 21vianet (CN) version, Google Drive.
 - Support for uploading files and folders to specified directories, keeping the directory structure as it was before the upload.
 - Supports the use of command parameters for external applications.
 - Support for customising the upload chunk size.
@@ -40,7 +40,7 @@ LightUploader -a "url" -v 2 -l zh-CN
 # This action will automatically initialise the configuration file
 
 # Google Drive
-LightUploader -a "url" -v 3
+LightUploader -v 3
 
 ```
 

googledrive/gdUpload.go

@@ -6,14 +6,19 @@ import (
 	"io/ioutil"
 	"log"
 	httpLocal "main/graph/net/http"
+	"math"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 	"unsafe"
 
+	"google.golang.org/api/googleapi"
+
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -22,6 +27,21 @@ import (
 
 // refs https://developers.google.com/drive/v3/web/quickstart/go
 
+var chunkSize = 10 * 1024 * 1024
+
+func changeChunkSize(block int) {
+	chunkSize = block * 1024 * 1024
+}
+
+var wg sync.WaitGroup
+var threads = 3
+var pool = make(chan struct{}, threads)
+
+func changeThread(thread int) {
+	threads = thread
+	pool = make(chan struct{}, threads)
+}
+
 // getClient uses a Context and Config to retrieve a Token
 // then generate a Client. It returns the generated Client.
 func getClient(file string, ctx context.Context, config *oauth2.Config) *http.Client {
@@ -82,7 +102,7 @@ func tokenCacheFile() string {
 
 // tokenFromFile retrieves a Token from a given file path.
 // It returns the retrieved Token and any read error encountered.
-func tokenFromFile(file string, Thread int, BlockSize int, Language string, TimeOut int, BotKey string, UserID string) (*oauth2.Token, error) {
+func tokenFromFile(c *oauth2.Config, file string, Thread int, BlockSize int, Language string, TimeOut int, BotKey string, UserID string) (*oauth2.Token, error) {
 	//搜尋路徑，有token檔就開啟並Decode，沒有就回傳nil
 	f, err := os.Open(file)
 	if err != nil {
@@ -92,14 +112,19 @@ func tokenFromFile(file string, Thread int, BlockSize int, Language string, Time
 	ts := &httpLocal.Certificate{}
 	err = json.NewDecoder(f).Decode(ts) //Decode錯誤也會回傳err
 	oauth2Token := ts.Other.(map[string]interface{})
-	expiry, _ := time.ParseInLocation("2021-04-29T16:27:54.2042271+08:00", oauth2Token["expiry"].(string), time.Local)
+	expiry, _ := time.Parse(time.RFC3339, oauth2Token["expiry"].(string))
 	t := &oauth2.Token{
 		AccessToken:  oauth2Token["access_token"].(string),
 		TokenType:    oauth2Token["token_type"].(string),
 		RefreshToken: oauth2Token["refresh_token"].(string),
 		Expiry:       expiry,
 	}
 	defer f.Close()
+	f, err = os.OpenFile(file, os.O_WRONLY|os.O_TRUNC, 0666)
+	defer f.Close()
+	// log.Printf("%+v\n", t)
+	updatedToken, err := c.TokenSource(context.TODO(), t).Token()
+	// log.Printf("%+v\n", updatedToken)
 	data := httpLocal.Certificate{
 		Drive:        "GoogleDrive",
 		RefreshToken: ts.RefreshToken,
@@ -110,17 +135,23 @@ func tokenFromFile(file string, Thread int, BlockSize int, Language string, Time
 		TimeOut:      TimeOut,
 		BotKey:       BotKey,
 		UserID:       UserID,
-		Other:        ts.Other,
+		Other:        updatedToken,
 	}
-	json.NewEncoder(f).Encode(data)
-	return t, err
+	err = json.NewEncoder(f).Encode(data)
+	if err != nil {
+		log.Fatalf("Unable to cache oauth token: %v", err)
+	}
+	changeThread(Thread)
+	changeChunkSize(BlockSize)
+	return updatedToken, err
 }
 
 // saveToken uses a file path to create a file and store the
 // token in it.
+
 func saveToken(file string, token *oauth2.Token, lang string) {
 	//fmt.Printf("Saving credential file to: %s\n", file)
-	f, err := os.Create(file)
+	f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
 	if err != nil {
 		log.Fatalf("Unable to cache oauth token: %v", err)
 	}
@@ -140,7 +171,46 @@ func saveToken(file string, token *oauth2.Token, lang string) {
 	json.NewEncoder(f).Encode(data)
 }
 
-func UploadAllFile(pathname string, folderIDList []string, srv *drive.Service, startTime int64, username string, sendMsg func(text string), locText func(text string) string) error {
+func byte2Readable(bytes float64) string {
+	const kb float64 = 1024
+	const mb float64 = kb * 1024
+	const gb float64 = mb * 1024
+	var readable float64
+	var unit string
+	_bytes := bytes
+
+	if _bytes >= gb {
+		// xx GB
+		readable = _bytes / gb
+		unit = "GB"
+	} else if _bytes < gb && _bytes >= mb {
+		// xx MB
+		readable = _bytes / mb
+		unit = "MB"
+	} else {
+		// xx KB
+		readable = _bytes / kb
+		unit = "KB"
+	}
+	return strconv.FormatFloat(readable, 'f', 2, 64) + " " + unit
+}
+
+func UploadAllFile(pathname string, folderIDList []string, srv *drive.Service, startTime int64, username string, sendMsg func() (func(text string), string, string, func(string, string, string) func(string)), locText func(text string) string) (func(string), error) {
+
+	filePath := path.Base(pathname)
+	temps, _botKey, iUserID, BotSend := sendMsg()
+	var iSendMsg func(string)
+	tip := "`" + filePath + "`" + locText("startUploadGoogleDrive")
+	if _botKey != "" && iUserID != "" {
+		iSendMsg = BotSend(_botKey, iUserID, tip)
+	}
+	temp := func(text string) {
+		temps(text)
+		if _botKey != "" && iUserID != "" {
+			iSendMsg(text)
+		}
+	}
+	temp(tip)
 
 	rd, err := ioutil.ReadDir(pathname)
 	if err != nil {
@@ -156,11 +226,9 @@ func UploadAllFile(pathname string, folderIDList []string, srv *drive.Service, s
 		defer f.Close()
 		//log.Println("file", fi.Name(), fullName)
 		//上傳檔案，create要給定檔案名稱，要傳進資料夾就加上Parents參數給定folderID的array，media傳入我們要上傳的檔案，最後Do
-		_, err = srv.Files.Create(&drive.File{Name: fullName, Parents: tempFolderIDList}).Media(f).Do()
-		if err != nil {
-			log.Panicf("Unable to create file: %v", err)
-		}
-		sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullName, time.Now().Unix()-startTime))
+
+		uploadFile(srv, pathname, fullName, tempFolderIDList, f, sendMsg, locText, username)
+		// sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullName, time.Now().Unix()-startTime))
 	}
 	_, foldName := filepath.Split(pathname)
 	//log.Println(foldName)
@@ -182,13 +250,13 @@ func UploadAllFile(pathname string, folderIDList []string, srv *drive.Service, s
 				tempFolderIDList = folderIDList
 			}
 			tempFolderIDList = append(tempFolderIDList, createFolder.Id)
-			err := UploadAllFile(fullDir, tempFolderIDList, srv, startTime, username, sendMsg, locText)
+			_, err := UploadAllFile(fullDir, tempFolderIDList, srv, startTime, username, sendMsg, locText)
 			if err != nil {
 				log.Panic("read dir fail:", err)
-				return err
+				return nil, err
 			}
 			//log.Println("folder", fi.Name(), fullDir)
-			sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullDir, time.Now().Unix()-startTime))
+			// sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullDir, time.Now().Unix()-startTime))
 		} else {
 			fullName := pathname + "/" + fi.Name()
 			var tempFolderIDList []string
@@ -203,15 +271,93 @@ func UploadAllFile(pathname string, folderIDList []string, srv *drive.Service, s
 			defer f.Close()
 			//log.Println("file", fi.Name(), fullName)
 			//上傳檔案，create要給定檔案名稱，要傳進資料夾就加上Parents參數給定folderID的array，media傳入我們要上傳的檔案，最後Do
-			_, err = srv.Files.Create(&drive.File{Name: fi.Name(), Parents: tempFolderIDList}).Media(f).Do()
+			// _, err = srv.Files.Create(&drive.File{Name: fi.Name(), Parents: tempFolderIDList}).Media(f, googleapi.ChunkSize(chunkSize)).Do()
+			uploadFile(srv, fullName, fi.Name(), tempFolderIDList, f, sendMsg, locText, username)
 			if err != nil {
 				log.Panicf("Unable to create file: %v", err)
 			}
-			sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullName, time.Now().Unix()-startTime))
+			// sendMsg(fmt.Sprintf(locText("googleDriveUploadTip"), username, fullName, time.Now().Unix()-startTime))
 			//log.Printf("file: %+v", driveFile)
 		}
 	}
-	return nil
+	wg.Wait()
+	return temp, nil
+}
+
+func FileSizeFormat(bytes int64, forceBytes bool) string {
+	if forceBytes {
+		return fmt.Sprintf("%v B", bytes)
+	}
+
+	units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
+
+	var i int
+	value := float64(bytes)
+
+	for value > 1000 {
+		value /= 1000
+		i++
+	}
+	return fmt.Sprintf("%.1f %s", value, units[i])
+}
+
+func MeasureTransferRate() func(int64) string {
+	start := time.Now()
+
+	return func(bytes int64) string {
+		seconds := int64(time.Now().Sub(start).Seconds())
+		if seconds < 1 {
+			return fmt.Sprintf("%s/s", FileSizeFormat(bytes, false))
+		}
+		bps := bytes / seconds
+		return fmt.Sprintf("%s/s", FileSizeFormat(bps, false))
+	}
+}
+
+func uploadFile(srv *drive.Service, filePath string, filename string, tempFolderIDList []string, f *os.File, sendMsg func() (func(text string), string, string, func(string, string, string) func(string)), locText func(text string) string, username string) {
+	wg.Add(1)
+	pool <- struct{}{}
+	startTime := time.Now().Unix()
+	go func() {
+		defer wg.Done()
+		defer func() {
+			<-pool
+		}()
+		temps, _botKey, iUserID, botSend := sendMsg()
+		var iSendMsg func(string)
+		tip := "`" + filePath + "`" + locText("startToUpload1")
+		if _botKey != "" && iUserID != "" {
+			iSendMsg = botSend(_botKey, iUserID, tip)
+		}
+		temp := func(text string) {
+			temps(text)
+			if _botKey != "" && iUserID != "" {
+				iSendMsg(text)
+			}
+		}
+		fi, err := os.Stat(filePath)
+		if err != nil {
+			fi, err = os.Stat(filename)
+			if err != nil {
+				log.Panicln(err)
+			}
+		}
+		getRate := MeasureTransferRate()
+		size := fi.Size()
+		// log.Println(byte2Readable(float64(size)))
+
+		showProgress := func(current, total int64) {
+			temp(fmt.Sprintf(locText("googleDriveUploadTip1"), username, filePath, byte2Readable(float64(size)), byte2Readable(float64(current)), int(math.Ceil(float64(current)/float64(chunkSize))), int(math.Ceil(float64(size)/float64(chunkSize))), getRate(current), time.Now().Unix()-startTime))
+		}
+
+		_, err = srv.Files.Create(&drive.File{Name: filename, Parents: tempFolderIDList}).Media(f, googleapi.ChunkSize(chunkSize)).ProgressUpdater(showProgress).Do()
+		if err != nil {
+			log.Panicln(err)
+		}
+
+		defer temp("close")
+	}()
+
 }
 
 /*func main() {
@@ -266,7 +412,8 @@ func CreateNewInfo(code string, lang string) string {
 
 func Upload(infoPath string, filePath string, sendMsg func() (func(text string), string, string, func(string, string, string) func(string)), locText func(text string) string, Thread int, BlockSize int, Language string, TimeOut int, BotKey string, UserID string) {
 	ctx, config := gdInit()
-	tok, _ := tokenFromFile(infoPath, Thread, BlockSize, Language, TimeOut, BotKey, UserID)
+	infoPath, _ = filepath.Abs(infoPath)
+	tok, _ := tokenFromFile(config, infoPath, Thread, BlockSize, Language, TimeOut, BotKey, UserID)
 	client := config.Client(ctx, tok)
 	srv, err := drive.New(client)
 	username := strings.ReplaceAll(filepath.Base(infoPath), ".json", "")
@@ -280,26 +427,12 @@ func Upload(infoPath string, filePath string, sendMsg func() (func(text string),
 	if err != nil {
 		log.Panic(err)
 	}
-	filePath = path.Base(filePath)
-	temps, _botKey, iUserID, BotSend := sendMsg()
-	var iSendMsg func(string)
-	tip := "`" + filePath + "`" + locText("startUploadGoogleDrive")
-	if _botKey != "" && iUserID != "" {
-		iSendMsg = BotSend(_botKey, iUserID, tip)
-	}
-	temp := func(text string) {
-		temps(text)
-		if _botKey != "" && iUserID != "" {
-			iSendMsg(text)
-		}
-	}
-	temp(tip)
 
 	if err != nil {
 		log.Fatalf("Unable to retrieve drive Client %v", err)
 
 	}
-	_ = UploadAllFile(filePath, nil, srv, time.Now().Unix(), username, temp, locText)
+	temp, _ := UploadAllFile(filePath, nil, srv, time.Now().Unix(), username, sendMsg, locText)
 	temp(locText("uploadGoogleDriveComplete"))
 	err = os.Chdir(oldDir)
 	if err != nil {

i18n.go

@@ -2,9 +2,6 @@ package main
 
 import (
 	"fmt"
-	"github.com/BurntSushi/toml"
-	"github.com/nicksnyder/go-i18n/v2/i18n"
-	"golang.org/x/text/language"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -13,6 +10,10 @@ import (
 	"path/filepath"
 	"regexp"
 	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/nicksnyder/go-i18n/v2/i18n"
+	"golang.org/x/text/language"
 )
 
 var bundle *i18n.Bundle
@@ -118,6 +119,7 @@ func (loc *Loc) init(locLanguage string) {
 		}
 	}
 	dir, err = os.Executable()
+	// log.Println(dir)
 	dropErr(err)
 	dir = filepath.Dir(dir)
 	rd, err := ioutil.ReadDir(dir)