first commit

.gitignore

@@ -0,0 +1,9 @@
+.history
+config.json
+temp.torrent
+.goreleaser.yml
+dist
+.idea
+info
+*.json
+*.exe

Readme.md

@@ -0,0 +1,135 @@
+# OneDriveUploader
+
+萌咖大佬写了一个[非常好的版本](https://github.com/MoeClub/OneList/tree/master/OneDriveUploader)，可惜并没有开源，而且已经好久都没有更新了。这个项目作为从[DownloadBot](https://github.com/gaowanliang/DownloadBot)中独立出来的一个简易上传工具，使得上传更加方便。同时会逐渐向萌咖的版本完善相关的功能，目前版本仅仅是从[DownloadBot](https://github.com/gaowanliang/DownloadBot)中独立的，只做了简单的功能。
+
+
+- 支持 国际版, 个人版(家庭版), ~~中国版(世纪互联)~~.
+- ~~支持上传文件和文件夹到指定目录,并保持上传前的目录结构.~~
+- 支持命令参数使用, 方便外部程序调用.
+- 支持自定义上传分块大小.
+- 支持多线程上传(多文件同时上传).
+- ~~支持根据文件大小动态调整重试次数~~
+- ~~支持跳过网盘中已存在的同名文件.~~
+- 支持通过Telegram Bot实时监控上传进度，方便使用全自动下载脚本时对上传的实时监控
+
+## 授权
+### 通过下面URL登录 (右键新标签打开)
+#### 国际版, 个人版(家庭版)
+[https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=ad5e65fd-856d-4356-aefc-537a9700c137&response_type=code&redirect_uri=http://localhost/onedrive-login&response_mode=query&scope=offline_access%20User.Read%20Files.ReadWrite.All](https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=ad5e65fd-856d-4356-aefc-537a9700c137&response_type=code&redirect_uri=http://localhost/onedrive-login&response_mode=query&scope=offline_access%20User.Read%20Files.ReadWrite.All)
+#### 中国版(世纪互联)
+
+
+### 初始化配置文件
+```bash
+# 国际版
+OneDriveUploader -a "url"
+
+# 个人版(家庭版)
+OneDriveUploader -a "url" -v 1
+# 中国版(世纪互联) 目前设计中，暂不可用
+OneDriveUploader -a "url" -v 2
+
+# 在浏览器地址栏中获取以 http://loaclhost 开头的整个url内容
+# 将获取的完整url内容替换命令中的 url 三个字母
+# 每次产生的 url 只能用一次, 重试请重新获取 url
+# 此操作将会自动初始化的配置文件
+```
+
+## 使用
+```bash
+Usage of OneDriveUploader:
+  -a string
+        // 初始化授权
+        Setup and Init auth.json.
+  -b string
+        // 自定义上传分块大小, 可以提高网络吞吐量, 受限于磁盘性能和网络速度.
+        Set block size. [Unit: M; 5<=b<=60;] (default "10")
+  -c string
+        // 配置文件路径
+        Config file. (default "auth.json")
+
+   //此参数未设计，暂不可用
+  -n string
+        // 上传单个文件时,在网盘中重命名
+        Rename file on upload to remote.
+
+  //此参数未设计，暂不可用
+  -r string
+        // 上传到网盘中的某个目录, 默认: 根目录
+        Upload to reomte path.
+
+  -f string
+        // *必要参数, 要上传的文件或文件夹
+        Upload item.
+  -t string
+        // 线程数, 同时上传文件的个数. 默认: 3
+        Set thread num. (default "3")
+
+  //此参数未设计，暂不可用
+  -force
+        // 开关(推荐)
+        // 加上 -f 参数，强制读取 auth.json 中的块大小配置和多线程配置.
+        // 不加 -f 参数, 每次覆盖保存当前使用参数到 auth.json 配置文件中.
+        Force Read config form config file. [BlockSize, ThreadNum]
+
+  //此参数未设计，暂不可用
+  -skip
+        // 开关
+        // 跳过上传网盘中已存在的同名文件. (默认不跳过)
+        Skip exist file on remote.
+
+```
+
+## 配置
+```json
+{
+    // 授权令牌
+    "RefreshToken": "1234564567890ABCDEF",
+    // 最大线程数.(同时上传文件的数量)
+    "ThreadNum": "2",
+    // 最大上传分块大小.(每次上传文件的最大分块大小,网络不好建议调低. 单位:MB)
+    "BlockSize": "10",
+    // 最大单文件大小.(目前: 个人版(家庭版)单文件限制为100GB; 其他版本单文件限制为15GB,微软将逐步更新为100GB. 单位:GB)
+    "SigleFile": "100",
+    // 缓存刷新间隔.
+    "RefreshInterval": 1500,
+    // 如果是中国版(世纪互联), 此项应为 true.
+    "MainLand": false
+}
+```
+
+## 示例
+```bash
+# 一些示例:
+
+# 将同目录下的 mm00.jpg 文件上传到 OneDrive 网盘根目录
+OneDriveUploader -c xxx.json -f "mm00.jpg"
+
+# 将同目录下的 mm00.jpg 文件上传到 OneDrive 网盘根目录,并改名为 mm01.jpg(暂不可用)
+OneDriveUploader  -c xxx.json -s "mm00.jpg" -n "mm01.jpg"
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录
+OneDriveUploader -c xxx.json -s "Download" 
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中(暂不可用)
+OneDriveUploader -c xxx.json -s "Download" -r "Test"
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程
+OneDriveUploader -c xxx.json -t 10 -s "Download" 
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程，同时使用 Telegram Bot 实时监控上传进度
+OneDriveUploader -c xxx.json -t 10 -s "Download" -tgbot "bot123456:xxxxxxxx" -uid 123456789
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 15 线程, 并设置分块大小为 20M
+OneDriveUploader -c xxx.json -t 15 -b 20 -s "Download" 
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中, 使用配置文件中的线程参数和分块大小参数(暂不可用)
+OneDriveUploader -f -c "/urs/local/auth.json" -s "Download" -r "Test"
+
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘Test目录中, 使用配置文件中的线程参数和分块大小参数，并跳过上传网盘中已存在的同名文件(暂不可用)
+OneDriveUploader -f -c "/urs/local/auth.json" -skip -s "Download" -r "Test"
+```
+
+## 注意
+- ~~多次尝试后, 无失败的上传文件. 退出码为 0 .~~
+- ~~最终还有失败的上传文件会详细列出上传失败项. 退出码为 1.~~

api/restore/upload/onedriveRecoverableRestore.go

@@ -0,0 +1,206 @@
+package upload
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"main/fileutil"
+	"runtime/debug"
+	"strconv"
+	"time"
+)
+
+const (
+	uploadSessionPath = "/users/%s/drive/root:/%s:/createUploadSession"
+	uploadURLKey      = "uploadUrl"
+)
+
+func (rs *RestoreService) recoverableUpload(userID string, bearerToken string, conflictOption string, filePath string, fileInfo fileutil.FileInfo, sendMsg func(text string), locText func(text string) string, username string) ([]map[string]interface{}, error) {
+	//1. Get recoverable upload session for the current file path 获取当前文件路径的可压缩上载会话
+	uploadSessionData, err := rs.getUploadSession(userID, bearerToken, conflictOption, filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	//2. Get the upload url returned as a response from the recoverable upload session above. 从上面的可压缩上载会话获取作为响应返回的上载url。
+	uploadURL := uploadSessionData[uploadURLKey].(string)
+
+	//3. Get the startOffset list for the file 获取文件的startOffset列表
+	startOffsetLst, err := fileutil.GetFileOffsetStash(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	//4. Loop over the file start offset list to read files in chunk and upload in onedrive 在文件开始偏移量列表上循环以读取块中的文件并在onedrive中上载
+	var uploadResp []map[string]interface{}
+	lastChunkIndex := len(startOffsetLst) - 1
+	var isLastChunk bool
+	timeUnix := time.Now().UnixNano()
+	var buffer = make([]byte, fileutil.GetDefaultChunkSize())
+	startTime := time.Now().Unix()
+	for i, sOffset := range startOffsetLst {
+		if i == lastChunkIndex {
+			lastChunkSize, err := fileutil.GetLatsChunkSizeInBytes(filePath)
+			if err != nil {
+				log.Panic(err)
+			}
+			buffer = make([]byte, lastChunkSize)
+			isLastChunk = true
+		}
+		filePartInBytes := &buffer
+		//4a. Get the bytes for the file based on the offset 根据偏移量获取文件的字节数
+		err := fileutil.GetFilePartInBytes(filePartInBytes, filePath, sOffset)
+		if err != nil {
+			return nil, err
+		}
+		if i != 0 {
+			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s` *『%d/%d』*  速度:`%s/s` 已耗时: `%d s`", username, filePath, i, len(startOffsetLst), byte2Readable(float64(fileutil.GetDefaultChunkSize())/float64(time.Now().UnixNano()-timeUnix)*float64(1000000000)), time.Now().Unix()-startTime))
+		} else {
+			sendMsg(fmt.Sprintf("正在向OneDrive账户 `%s` 上传 `%s` *『%d/%d』*  速度:`----` 已耗时: `%d s`", username, filePath, i, len(startOffsetLst), time.Now().Unix()-startTime))
+		}
+
+		timeUnix = time.Now().UnixNano()
+		//3b. make a call to the upload url with the file part based on the offset. 使用基于偏移量的文件部分调用上载url。
+		resp, err := rs.uploadFilePart(uploadURL, filePath, bearerToken, *filePartInBytes, sOffset, isLastChunk)
+
+		if err != nil {
+			return nil, err
+		}
+		respMap := make(map[string]interface{})
+		err = json.NewDecoder(resp.Body).Decode(&respMap)
+		if err != nil {
+			fmt.Println(err)
+		}
+		if resp.Body != nil {
+			defer resp.Body.Close()
+		}
+		//fmt.Printf("%+v, status code: %s", respMap, resp.Status)
+		uploadResp = append(uploadResp, respMap)
+		debug.FreeOSMemory()
+	}
+	sendMsg("close")
+	return uploadResp, nil
+}
+
+//Returns the restore session url for part file upload
+func (rs *RestoreService) getUploadSession(userID string, bearerToken string, conflictOption string, filePath string) (map[string]interface{}, error) {
+	uploadSessionPath := fmt.Sprintf(uploadSessionPath, userID, filePath)
+	uploadSessionData := make(map[string]interface{})
+	//Get the body for resemble upload session call.
+	body, err := getRessumableSessionBody(filePath, conflictOption)
+	if err != nil {
+		return nil, err
+	}
+
+	//Create request instance
+	req, err := rs.NewRequest("POST", uploadSessionPath, getRessumableUploadSessionHeader(bearerToken), body)
+	if err != nil {
+		return nil, err
+	}
+	//Execute the request
+	resp, err := rs.Do(req)
+	if err != nil {
+		//Need to return a generic object from onedrive upload instead of response directly
+		return nil, err
+	}
+
+	//convert http.Response to map
+	err = json.NewDecoder(resp.Body).Decode(&uploadSessionData)
+	if err != nil {
+		return nil, err
+	}
+	return uploadSessionData, nil
+}
+
+//Uploads the file part to Onedrive
+func (rs *RestoreService) uploadFilePart(uploadURL string, filePath string, bearerToken string, filePart []byte, startOffset int64, isLastPart bool) (*http.Response, error) {
+	//This is required for Content-Range header key
+	fileSizeInBytes, err := fileutil.GetFileSize(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	//Fetch Last chunklength -- will be needed in Content_length header
+	lastChunkLength, err := fileutil.GetLatsChunkSizeInBytes(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	//Create upload part file request
+	req, err := rs.NewRequest("PUT", uploadURL, getRessumableUploadHeader(fileSizeInBytes, bearerToken, startOffset, isLastPart, lastChunkLength), filePart)
+	if err != nil {
+		return nil, err
+	}
+
+	//Execute the request
+	resp, err := rs.Do(req)
+	if err != nil {
+		//Need to return a generic object from onedrive upload instead of response directly
+		return nil, err
+	}
+	return resp, nil
+}
+
+//Returns header for upload session API
+func getRessumableUploadSessionHeader(accessToken string) map[string]string {
+	//As a work around for now, ultimately this will be recived as a part of restore xml
+	bearerToken := fmt.Sprintf("bearer %s", accessToken)
+	return map[string]string{
+		"Content-Type":  "application/json",
+		"Authorization": bearerToken,
+	}
+}
+
+//Returns headers for recoverable actual upload as file parts
+func getRessumableUploadHeader(fileSizeInBytes int64, accessToken string, startOffset int64, isLastChunk bool, lastChunkSize int64) map[string]string {
+	var cRange string
+	var cLength string
+
+	if isLastChunk {
+		cRange = fmt.Sprintf("bytes %d-%d/%d", startOffset, fileSizeInBytes-2, fileSizeInBytes-1)
+		cLength = fmt.Sprintf("%d", lastChunkSize)
+	} else {
+		cRange = fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+fileutil.GetDefaultChunkSize()-1, fileSizeInBytes-1)
+		cLength = fmt.Sprintf("%d", fileutil.GetDefaultChunkSize())
+	}
+
+	// fmt.Printf("\nCLength: %s , cRange: %s\n", cLength, cRange)
+	bearerToken := fmt.Sprintf("bearer %s", accessToken)
+	return map[string]string{
+		"Content-Length": cLength,
+		"Content-Range":  cRange,
+		"Authorization":  bearerToken,
+	}
+}
+
+//Returns the expected body for creating file upload session to onedrive
+func getRessumableSessionBody(filePath string, conflictOption string) (string, error) {
+	bodyMap := map[string]string{"@microsoft.graph.conflictBehavior": conflictOption, "description": "", "name": filePath}
+	jsonBody, err := json.Marshal(bodyMap)
+	return string(jsonBody), err
+}
+
+func byte2Readable(bytes float64) string {
+	const kb float64 = 1024
+	const mb float64 = kb * 1024
+	const gb float64 = mb * 1024
+	var readable float64
+	var unit string
+	_bytes := bytes
+
+	if _bytes >= gb {
+		// xx GB
+		readable = _bytes / gb
+		unit = "GB"
+	} else if _bytes < gb && _bytes >= mb {
+		// xx MB
+		readable = _bytes / mb
+		unit = "MB"
+	} else {
+		// xx KB
+		readable = _bytes / kb
+		unit = "KB"
+	}
+	return strconv.FormatFloat(readable, 'f', 2, 64) + " " + unit
+}