From 10191a1be37b02f1a4a643bc45fed5940b85327f Mon Sep 17 00:00:00 2001 From: link Date: Fri, 17 Mar 2023 15:37:28 +0800 Subject: [PATCH] Test rclone (#961) --- cmd/migration-tool/migration_042.go | 67 ---- cmd/migration-tool/migration_dummy.go | 27 ++ drivers/all.go | 12 + drivers/base/client.go | 30 ++ drivers/base/types.go | 12 + drivers/dropbox/drive.go | 100 +++++ drivers/dropbox/meta.go | 33 ++ drivers/dropbox/types.go | 88 +++++ drivers/dropbox/util.go | 102 +++++ drivers/google_drive/drive.go | 183 +++++++++ drivers/google_drive/meta.go | 35 ++ drivers/google_drive/types.go | 77 ++++ drivers/google_drive/util.go | 152 +++++++ internal/conf/config.go | 43 ++ internal/conf/const.go | 72 ++++ internal/conf/var.go | 30 ++ internal/driver/config.go | 25 ++ internal/driver/driver.go | 131 +++++++ internal/driver/item.go | 56 +++ internal/op/const.go | 6 + internal/op/driver.go | 173 ++++++++ internal/op/fs.go | 545 ++++++++++++++++++++++++++ internal/op/hook.go | 109 ++++++ internal/sign/sign.go | 36 ++ main.go | 12 +- model/storage.go | 8 +- pkg/utils/httper/drive.go | 255 ++++++------ route/v1.go | 13 +- route/v1/cloud.go | 101 +++++ route/v1/driver.go | 12 + route/v1/recover.go | 205 ++++++++++ service/service.go | 15 +- service/storage.go | 116 ++++++ service/storage_old.go | 73 ---- service/system.go | 9 +- 35 files changed, 2685 insertions(+), 278 deletions(-) delete mode 100644 cmd/migration-tool/migration_042.go create mode 100644 cmd/migration-tool/migration_dummy.go create mode 100644 drivers/all.go create mode 100644 drivers/base/client.go create mode 100644 drivers/base/types.go create mode 100644 drivers/dropbox/drive.go create mode 100644 drivers/dropbox/meta.go create mode 100644 drivers/dropbox/types.go create mode 100644 drivers/dropbox/util.go create mode 100644 drivers/google_drive/drive.go create mode 100644 drivers/google_drive/meta.go create mode 100644 drivers/google_drive/types.go create mode 100644 drivers/google_drive/util.go create mode 100644 internal/conf/config.go create mode 100644 internal/conf/const.go create mode 100644 internal/conf/var.go create mode 100644 internal/driver/config.go create mode 100644 internal/driver/driver.go create mode 100644 internal/driver/item.go create mode 100644 internal/op/const.go create mode 100644 internal/op/driver.go create mode 100644 internal/op/fs.go create mode 100644 internal/op/hook.go create mode 100644 internal/sign/sign.go create mode 100644 route/v1/cloud.go create mode 100644 route/v1/driver.go create mode 100644 route/v1/recover.go create mode 100644 service/storage.go delete mode 100644 service/storage_old.go diff --git a/cmd/migration-tool/migration_042.go b/cmd/migration-tool/migration_042.go deleted file mode 100644 index e8abd4464..000000000 --- a/cmd/migration-tool/migration_042.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * @Author: LinkLeong link@icewhale.org - * @Date: 2022-08-24 17:36:00 - * @LastEditors: LinkLeong - * @LastEditTime: 2022-09-05 11:24:27 - * @FilePath: /CasaOS/cmd/migration-tool/migration-034-035.go - * @Description: - * @Website: https://www.casaos.io - * Copyright (c) 2022 by icewhale, All Rights Reserved. - */ -package main - -import ( - "os" - - interfaces "github.com/IceWhaleTech/CasaOS-Common" - "github.com/IceWhaleTech/CasaOS-Common/utils/version" - "github.com/IceWhaleTech/CasaOS/pkg/utils/command" -) - -type migrationTool struct{} - -func (u *migrationTool) IsMigrationNeeded() (bool, error) { - majorVersion, minorVersion, patchVersion, err := version.DetectLegacyVersion() - if err != nil { - if err == version.ErrLegacyVersionNotFound { - return false, nil - } - - return false, err - } - - if majorVersion > 0 { - return false, nil - } - - if minorVersion > 4 { - return false, nil - } - - if minorVersion == 4 && patchVersion != 2 { - return false, nil - } - - _logger.Info("Migration is needed for a CasaOS version 0.4.2 ") - return true, nil -} - -func (u *migrationTool) PreMigrate() error { - return nil -} - -func (u *migrationTool) Migrate() error { - _logger.Info("Migration is started for a CasaOS version 0.4.2 ") - command.OnlyExec("systemctl stop rclone.service") - os.Remove("/usr/lib/systemd/system/rclone.service") - command.OnlyExec("systemctl daemon-reload") - return nil -} - -func (u *migrationTool) PostMigrate() error { - return nil -} - -func NewMigrationDummy() interfaces.MigrationTool { - return &migrationTool{} -} diff --git a/cmd/migration-tool/migration_dummy.go b/cmd/migration-tool/migration_dummy.go new file mode 100644 index 000000000..6d08df255 --- /dev/null +++ b/cmd/migration-tool/migration_dummy.go @@ -0,0 +1,27 @@ +package main + +import ( + interfaces "github.com/IceWhaleTech/CasaOS-Common" +) + +type migrationTool struct{} + +func (u *migrationTool) IsMigrationNeeded() (bool, error) { + return false, nil +} + +func (u *migrationTool) PreMigrate() error { + return nil +} + +func (u *migrationTool) Migrate() error { + return nil +} + +func (u *migrationTool) PostMigrate() error { + return nil +} + +func NewMigrationDummy() interfaces.MigrationTool { + return &migrationTool{} +} diff --git a/drivers/all.go b/drivers/all.go new file mode 100644 index 000000000..3cdfe8433 --- /dev/null +++ b/drivers/all.go @@ -0,0 +1,12 @@ +package drivers + +import ( + _ "github.com/IceWhaleTech/CasaOS/drivers/dropbox" + _ "github.com/IceWhaleTech/CasaOS/drivers/google_drive" +) + +// All do nothing,just for import +// same as _ import +func All() { + +} diff --git a/drivers/base/client.go b/drivers/base/client.go new file mode 100644 index 000000000..02e314b24 --- /dev/null +++ b/drivers/base/client.go @@ -0,0 +1,30 @@ +package base + +import ( + "net/http" + "time" + + "github.com/go-resty/resty/v2" +) + +var NoRedirectClient *resty.Client +var RestyClient = NewRestyClient() +var HttpClient = &http.Client{} +var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" +var DefaultTimeout = time.Second * 30 + +func init() { + NoRedirectClient = resty.New().SetRedirectPolicy( + resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }), + ) + NoRedirectClient.SetHeader("user-agent", UserAgent) +} + +func NewRestyClient() *resty.Client { + return resty.New(). + SetHeader("user-agent", UserAgent). + SetRetryCount(3). + SetTimeout(DefaultTimeout) +} diff --git a/drivers/base/types.go b/drivers/base/types.go new file mode 100644 index 000000000..e2757f217 --- /dev/null +++ b/drivers/base/types.go @@ -0,0 +1,12 @@ +package base + +import "github.com/go-resty/resty/v2" + +type Json map[string]interface{} + +type TokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` +} + +type ReqCallback func(req *resty.Request) diff --git a/drivers/dropbox/drive.go b/drivers/dropbox/drive.go new file mode 100644 index 000000000..fde830a2b --- /dev/null +++ b/drivers/dropbox/drive.go @@ -0,0 +1,100 @@ +package dropbox + +import ( + "context" + "errors" + "net/http" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" +) + +type Dropbox struct { + model.StorageA + Addition + AccessToken string +} + +func (d *Dropbox) Config() driver.Config { + return config +} + +func (d *Dropbox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Dropbox) Init(ctx context.Context) error { + if len(d.RefreshToken) == 0 { + d.getRefreshToken() + } + return d.refreshToken() +} + +func (d *Dropbox) Drop(ctx context.Context) error { + + return nil +} + +func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := "https://content.dropboxapi.com/2/files/download" + link := model.Link{ + URL: url, + Method: http.MethodPost, + Header: http.Header{ + "Authorization": []string{"Bearer " + d.AccessToken}, + "Dropbox-API-Arg": []string{`{"path": "` + file.GetPath() + `"}`}, + }, + } + return &link, nil +} +func (d *Dropbox) GetUserInfo(ctx context.Context) (string, error) { + url := "https://api.dropboxapi.com/2/users/get_current_account" + user := UserInfo{} + resp, err := d.request(url, http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "") + }, &user) + if err != nil { + return "", err + } + logger.Info("resp", zap.Any("resp", string(resp))) + return user.Email, nil +} +func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return nil +} + +func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return nil +} + +func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return nil +} + +func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errors.New("not support") +} + +func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error { + return nil +} + +func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return nil +} + +var _ driver.Driver = (*Dropbox)(nil) diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.go new file mode 100644 index 000000000..24d535d12 --- /dev/null +++ b/drivers/dropbox/meta.go @@ -0,0 +1,33 @@ +package dropbox + +import ( + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" +) + +const ICONURL = "./img/driver/Dropbox.svg" +const APPKEY = "tciqajyazzdygt9" +const APPSECRET = "e7gtmv441cwdf0n" + +type Addition struct { + driver.RootID + RefreshToken string `json:"refresh_token" required:"true" omit:"true"` + AppKey string `json:"app_key" type:"string" default:"tciqajyazzdygt9" omit:"true"` + AppSecret string `json:"app_secret" type:"string" default:"e7gtmv441cwdf0n" omit:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"` + AuthUrl string `json:"auth_url" type:"string" default:"https://www.dropbox.com/oauth2/authorize?client_id=tciqajyazzdygt9&redirect_uri=https://cloudoauth.files.casaos.app&response_type=code&token_access_type=offline&state=${HOST}%2Fv1%2Frecover%2FDropbox&&force_reapprove=true&force_reauthentication=true"` + Icon string `json:"icon" type:"string" default:"./img/driver/Dropbox.svg"` + Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"` +} + +var config = driver.Config{ + Name: "Dropbox", + OnlyProxy: true, + DefaultRoot: "root", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Dropbox{} + }) +} diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.go new file mode 100644 index 000000000..af5bdb9b1 --- /dev/null +++ b/drivers/dropbox/types.go @@ -0,0 +1,88 @@ +package dropbox + +import ( + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/model" + "go.uber.org/zap" +) + +type UserInfo struct { + AccountID string `json:"account_id"` + Name struct { + GivenName string `json:"given_name"` + Surname string `json:"surname"` + FamiliarName string `json:"familiar_name"` + DisplayName string `json:"display_name"` + AbbreviatedName string `json:"abbreviated_name"` + } `json:"name"` + Email string `json:"email"` + EmailVerified bool `json:"email_verified"` + Disabled bool `json:"disabled"` + Country string `json:"country"` + Locale string `json:"locale"` + ReferralLink string `json:"referral_link"` + IsPaired bool `json:"is_paired"` + AccountType struct { + Tag string `json:".tag"` + } `json:"account_type"` + RootInfo struct { + Tag string `json:".tag"` + RootNamespaceID string `json:"root_namespace_id"` + HomeNamespaceID string `json:"home_namespace_id"` + } `json:"root_info"` +} +type TokenError struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} +type File struct { + Tag string `json:".tag"` + Name string `json:"name"` + PathLower string `json:"path_lower"` + PathDisplay string `json:"path_display"` + ID string `json:"id"` + ClientModified time.Time `json:"client_modified,omitempty"` + ServerModified time.Time `json:"server_modified,omitempty"` + Rev string `json:"rev,omitempty"` + Size int `json:"size,omitempty"` + IsDownloadable bool `json:"is_downloadable,omitempty"` + ContentHash string `json:"content_hash,omitempty"` +} + +type Files struct { + Files []File `json:"entries"` + Cursor string `json:"cursor"` + HasMore bool `json:"has_more"` +} + +type Error struct { + Error struct { + Errors []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + } + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} + +func fileToObj(f File) *model.ObjThumb { + logger.Info("dropbox file", zap.Any("file", f)) + obj := &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + Name: f.Name, + Size: int64(f.Size), + Modified: f.ClientModified, + IsFolder: f.Tag == "folder", + Path: f.PathDisplay, + }, + Thumbnail: model.Thumbnail{}, + } + return obj +} diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go new file mode 100644 index 000000000..7cdf6de52 --- /dev/null +++ b/drivers/dropbox/util.go @@ -0,0 +1,102 @@ +package dropbox + +import ( + "fmt" + "net/http" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" +) + +func (d *Dropbox) getRefreshToken() error { + url := "https://api.dropbox.com/oauth2/token" + var resp base.TokenResp + var e TokenError + + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "code": d.Code, + "grant_type": "authorization_code", + "redirect_uri": "https://cloudoauth.files.casaos.app", + }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.RefreshToken = resp.RefreshToken + return nil + +} +func (d *Dropbox) refreshToken() error { + url := "https://api.dropbox.com/oauth2/token" + var resp base.TokenResp + var e TokenError + + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "refresh_token": d.RefreshToken, + "grant_type": "refresh_token", + }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.AccessToken = resp.AccessToken + return nil + +} +func (d *Dropbox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeader("Content-Type", "application/json") + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Error + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } + return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + return res.Body(), nil +} +func (d *Dropbox) getFiles(path string) ([]File, error) { + + res := make([]File, 0) + var resp Files + body := base.Json{ + "limit": 2000, + "path": path, + } + + _, err := d.request("https://api.dropboxapi.com/2/files/list_folder", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp) + if err != nil { + return nil, err + } + res = append(res, resp.Files...) + + return res, nil +} diff --git a/drivers/google_drive/drive.go b/drivers/google_drive/drive.go new file mode 100644 index 000000000..72babbd91 --- /dev/null +++ b/drivers/google_drive/drive.go @@ -0,0 +1,183 @@ +package google_drive + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" +) + +type GoogleDrive struct { + model.StorageA + Addition + AccessToken string +} + +func (d *GoogleDrive) Config() driver.Config { + return config +} + +func (d *GoogleDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GoogleDrive) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = 5 + } + if len(d.RefreshToken) == 0 { + d.getRefreshToken() + } + return d.refreshToken() +} + +func (d *GoogleDrive) Drop(ctx context.Context) error { + return nil +} + +func (d *GoogleDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?includeItemsFromAllDrives=true&supportsAllDrives=true", file.GetID()) + _, err := d.request(url, http.MethodGet, nil, nil) + if err != nil { + return nil, err + } + link := model.Link{ + Method: http.MethodGet, + URL: url + "&alt=media", + Header: http.Header{ + "Authorization": []string{"Bearer " + d.AccessToken}, + }, + } + return &link, nil +} +func (d *GoogleDrive) GetUserInfo(ctx context.Context) (string, error) { + url := "https://content.googleapis.com/drive/v3/about?fields=user" + user := UserInfo{} + resp, err := d.request(url, http.MethodGet, nil, &user) + if err != nil { + return "", err + } + logger.Info("resp", zap.Any("resp", resp)) + return user.User.EmailAddress, nil +} + +func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + data := base.Json{ + "name": dirName, + "parents": []string{parentDir.GetID()}, + "mimeType": "application/vnd.google-apps.folder", + } + _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return err +} + +func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + query := map[string]string{ + "addParents": dstDir.GetID(), + "removeParents": "root", + } + url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() + _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + return err +} + +func (d *GoogleDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + data := base.Json{ + "name": newName, + } + url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() + _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return err +} + +func (d *GoogleDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errors.New("not support") +} + +func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { + url := "https://www.googleapis.com/drive/v3/files/" + obj.GetID() + _, err := d.request(url, http.MethodDelete, nil, nil) + return err +} + +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + obj := stream.GetOld() + var ( + e Error + url string + data base.Json + res *resty.Response + err error + ) + if obj != nil { + url = fmt.Sprintf("https://www.googleapis.com/upload/drive/v3/files/%s?uploadType=resumable&supportsAllDrives=true", obj.GetID()) + data = base.Json{} + } else { + data = base.Json{ + "name": stream.GetName(), + "parents": []string{dstDir.GetID()}, + } + url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" + } + req := base.NoRedirectClient.R(). + SetHeaders(map[string]string{ + "Authorization": "Bearer " + d.AccessToken, + "X-Upload-Content-Type": stream.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + }). + SetError(&e).SetBody(data).SetContext(ctx) + if obj != nil { + res, err = req.Patch(url) + } else { + res, err = req.Post(url) + } + if err != nil { + return err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return err + } + return d.Put(ctx, dstDir, stream, up) + } + return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + putUrl := res.Header().Get("location") + if stream.GetSize() < d.ChunkSize*1024*1024 { + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)).SetBody(stream.GetReadCloser()) + }, nil) + } else { + err = d.chunkUpload(ctx, stream, putUrl) + } + return err +} + +var _ driver.Driver = (*GoogleDrive)(nil) diff --git a/drivers/google_drive/meta.go b/drivers/google_drive/meta.go new file mode 100644 index 000000000..f77196676 --- /dev/null +++ b/drivers/google_drive/meta.go @@ -0,0 +1,35 @@ +package google_drive + +import ( + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" +) + +const ICONURL = "./img/driver/GoogleDrive.svg" +const CLIENTID = "921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com" +const CLIENTSECRET = "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" + +type Addition struct { + driver.RootID + RefreshToken string `json:"refresh_token" required:"true" omit:"true"` + OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime" omit:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"` + ClientID string `json:"client_id" required:"true" default:"921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com" omit:"true"` + ClientSecret string `json:"client_secret" required:"true" default:"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" omit:"true"` + ChunkSize int64 `json:"chunk_size" type:"number" help:"chunk size while uploading (unit: MB)" omit:"true"` + AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com&redirect_uri=https%3A%2F%2Fcloudoauth.files.casaos.app&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"` + Icon string `json:"icon" type:"string" default:"./img/driver/GoogleDrive.svg"` + Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"` +} + +var config = driver.Config{ + Name: "GoogleDrive", + OnlyProxy: true, + DefaultRoot: "root", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GoogleDrive{} + }) +} diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go new file mode 100644 index 000000000..4bbab52b8 --- /dev/null +++ b/drivers/google_drive/types.go @@ -0,0 +1,77 @@ +package google_drive + +import ( + "strconv" + "time" + + "github.com/IceWhaleTech/CasaOS/model" + log "github.com/sirupsen/logrus" +) + +type UserInfo struct { + User struct { + Kind string `json:"kind"` + DisplayName string `json:"displayName"` + PhotoLink string `json:"photoLink"` + Me bool `json:"me"` + PermissionID string `json:"permissionId"` + EmailAddress string `json:"emailAddress"` + } `json:"user"` +} + +type TokenError struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +type Files struct { + NextPageToken string `json:"nextPageToken"` + Files []File `json:"files"` +} + +type File struct { + Id string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mimeType"` + ModifiedTime time.Time `json:"modifiedTime"` + Size string `json:"size"` + ThumbnailLink string `json:"thumbnailLink"` + ShortcutDetails struct { + TargetId string `json:"targetId"` + TargetMimeType string `json:"targetMimeType"` + } `json:"shortcutDetails"` +} + +func fileToObj(f File) *model.ObjThumb { + log.Debugf("google file: %+v", f) + size, _ := strconv.ParseInt(f.Size, 10, 64) + obj := &model.ObjThumb{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: size, + Modified: f.ModifiedTime, + IsFolder: f.MimeType == "application/vnd.google-apps.folder", + }, + Thumbnail: model.Thumbnail{}, + } + if f.MimeType == "application/vnd.google-apps.shortcut" { + obj.ID = f.ShortcutDetails.TargetId + obj.IsFolder = f.ShortcutDetails.TargetMimeType == "application/vnd.google-apps.folder" + } + return obj +} + +type Error struct { + Error struct { + Errors []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + } + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go new file mode 100644 index 000000000..a3b443af2 --- /dev/null +++ b/drivers/google_drive/util.go @@ -0,0 +1,152 @@ +package google_drive + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "go.uber.org/zap" +) + +// do others that not defined in Driver interface + +func (d *GoogleDrive) getRefreshToken() error { + url := "https://www.googleapis.com/oauth2/v4/token" + var resp base.TokenResp + var e TokenError + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + "code": d.Code, + "grant_type": "authorization_code", + "redirect_uri": "https://cloudoauth.files.casaos.app", + }).Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.RefreshToken = resp.RefreshToken + return nil +} + +func (d *GoogleDrive) refreshToken() error { + url := "https://www.googleapis.com/oauth2/v4/token" + var resp base.TokenResp + var e TokenError + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + "refresh_token": d.RefreshToken, + "grant_type": "refresh_token", + }).Post(url) + if err != nil { + return err + } + log.Debug(res.String()) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.AccessToken = resp.AccessToken + return nil +} + +func (d *GoogleDrive) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetQueryParam("includeItemsFromAllDrives", "true") + req.SetQueryParam("supportsAllDrives", "true") + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Error + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } + return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + return res.Body(), nil +} + +func (d *GoogleDrive) getFiles(id string) ([]File, error) { + pageToken := "first" + res := make([]File, 0) + for pageToken != "" { + if pageToken == "first" { + pageToken = "" + } + var resp Files + orderBy := "folder,name,modifiedTime desc" + if d.OrderBy != "" { + orderBy = d.OrderBy + " " + d.OrderDirection + } + query := map[string]string{ + "orderBy": orderBy, + "fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken", + "pageSize": "1000", + "q": fmt.Sprintf("'%s' in parents and trashed = false", id), + //"includeItemsFromAllDrives": "true", + //"supportsAllDrives": "true", + "pageToken": pageToken, + } + _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + pageToken = resp.NextPageToken + res = append(res, resp.Files...) + } + return res, nil +} + +func (d *GoogleDrive) chunkUpload(ctx context.Context, stream model.FileStreamer, url string) error { + var defaultChunkSize = d.ChunkSize * 1024 * 1024 + var finish int64 = 0 + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + chunkSize := stream.GetSize() - finish + if chunkSize > defaultChunkSize { + chunkSize = defaultChunkSize + } + _, err := d.request(url, http.MethodPut, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Length": strconv.FormatInt(chunkSize, 10), + "Content-Range": fmt.Sprintf("bytes %d-%d/%d", finish, finish+chunkSize-1, stream.GetSize()), + }).SetBody(io.LimitReader(stream.GetReadCloser(), chunkSize)).SetContext(ctx) + }, nil) + if err != nil { + return err + } + finish += chunkSize + } + return nil +} diff --git a/internal/conf/config.go b/internal/conf/config.go new file mode 100644 index 000000000..16dabae2f --- /dev/null +++ b/internal/conf/config.go @@ -0,0 +1,43 @@ +package conf + +type Database struct { + Type string `json:"type" env:"DB_TYPE"` + Host string `json:"host" env:"DB_HOST"` + Port int `json:"port" env:"DB_PORT"` + User string `json:"user" env:"DB_USER"` + Password string `json:"password" env:"DB_PASS"` + Name string `json:"name" env:"DB_NAME"` + DBFile string `json:"db_file" env:"DB_FILE"` + TablePrefix string `json:"table_prefix" env:"DB_TABLE_PREFIX"` + SSLMode string `json:"ssl_mode" env:"DB_SSL_MODE"` +} + +type Scheme struct { + Https bool `json:"https" env:"HTTPS"` + CertFile string `json:"cert_file" env:"CERT_FILE"` + KeyFile string `json:"key_file" env:"KEY_FILE"` +} + +type LogConfig struct { + Enable bool `json:"enable" env:"LOG_ENABLE"` + Name string `json:"name" env:"LOG_NAME"` + MaxSize int `json:"max_size" env:"MAX_SIZE"` + MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` + MaxAge int `json:"max_age" env:"MAX_AGE"` + Compress bool `json:"compress" env:"COMPRESS"` +} + +type Config struct { + Force bool `json:"force" env:"FORCE"` + Address string `json:"address" env:"ADDR"` + Port int `json:"port" env:"PORT"` + SiteURL string `json:"site_url" env:"SITE_URL"` + Cdn string `json:"cdn" env:"CDN"` + JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` + TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` + Database Database `json:"database"` + Scheme Scheme `json:"scheme"` + TempDir string `json:"temp_dir" env:"TEMP_DIR"` + BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` + Log LogConfig `json:"log"` +} diff --git a/internal/conf/const.go b/internal/conf/const.go new file mode 100644 index 000000000..6a75634ef --- /dev/null +++ b/internal/conf/const.go @@ -0,0 +1,72 @@ +package conf + +const ( + TypeString = "string" + TypeSelect = "select" + TypeBool = "bool" + TypeText = "text" + TypeNumber = "number" +) + +const ( + // site + VERSION = "version" + ApiUrl = "api_url" + BasePath = "base_path" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + + Logo = "logo" + Favicon = "favicon" + MainColor = "main_color" + + // preview + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + + // global + HideFiles = "hide_files" + CustomizeHead = "customize_head" + CustomizeBody = "customize_body" + LinkExpiration = "link_expiration" + SignAll = "sign_all" + PrivacyRegs = "privacy_regs" + OcrApi = "ocr_api" + FilenameCharMapping = "filename_char_mapping" + + // index + SearchIndex = "search_index" + AutoUpdateIndex = "auto_update_index" + IndexPaths = "index_paths" + IgnorePaths = "ignore_paths" + + // aria2 + Aria2Uri = "aria2_uri" + Aria2Secret = "aria2_secret" + + // single + Token = "token" + IndexProgress = "index_progress" + + //Github + GithubClientId = "github_client_id" + GithubClientSecrets = "github_client_secrets" + GithubLoginEnabled = "github_login_enabled" +) + +const ( + UNKNOWN = iota + FOLDER + //OFFICE + VIDEO + AUDIO + TEXT + IMAGE +) diff --git a/internal/conf/var.go b/internal/conf/var.go new file mode 100644 index 000000000..a91547d11 --- /dev/null +++ b/internal/conf/var.go @@ -0,0 +1,30 @@ +package conf + +import "regexp" + +var ( + BuiltAt string + GoVersion string + GitAuthor string + GitCommit string + Version string = "dev" + WebVersion string +) + +var ( + Conf *Config +) + +var SlicesMap = make(map[string][]string) +var FilenameCharMap = make(map[string]string) +var PrivacyReg []*regexp.Regexp + +var ( + // StoragesLoaded loaded success if empty + StoragesLoaded = false +) +var ( + RawIndexHtml string + ManageHtml string + IndexHtml string +) diff --git a/internal/driver/config.go b/internal/driver/config.go new file mode 100644 index 000000000..a7758ba30 --- /dev/null +++ b/internal/driver/config.go @@ -0,0 +1,25 @@ +/* + * @Author: a624669980@163.com a624669980@163.com + * @Date: 2022-12-13 11:05:05 + * @LastEditors: a624669980@163.com a624669980@163.com + * @LastEditTime: 2022-12-13 11:05:13 + * @FilePath: /drive/internal/driver/config.go + * @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + */ +package driver + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` // if need get message from user, such as validate code + DefaultRoot string `json:"default_root"` + CheckStatus bool +} + +func (c Config) MustProxy() bool { + return c.OnlyProxy || c.OnlyLocal +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go new file mode 100644 index 000000000..7519b5e5d --- /dev/null +++ b/internal/driver/driver.go @@ -0,0 +1,131 @@ +package driver + +import ( + "context" + + "github.com/IceWhaleTech/CasaOS/model" +) + +type Driver interface { + Meta + Reader + User + //Writer + //Other +} + +type Meta interface { + Config() Config + // GetStorage just get raw storage, no need to implement, because model.Storage have implemented + GetStorage() *model.StorageA + SetStorage(model.StorageA) + // GetAddition Additional is used for unmarshal of JSON, so need return pointer + GetAddition() Additional + // Init If already initialized, drop first + Init(ctx context.Context) error + Drop(ctx context.Context) error +} + +type Other interface { + Other(ctx context.Context, args model.OtherArgs) (interface{}, error) +} + +type Reader interface { + // List files in the path + // if identify files by path, need to set ID with path,like path.Join(dir.GetID(), obj.GetName()) + // if identify files by id, need to set ID with corresponding id + List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) + // Link get url/filepath/reader of file + Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) +} +type User interface { + // GetRoot get root directory of user + GetUserInfo(ctx context.Context) (string, error) +} +type Getter interface { + GetRoot(ctx context.Context) (model.Obj, error) +} + +//type Writer interface { +// Mkdir +// Move +// Rename +// Copy +// Remove +// Put +//} + +type Mkdir interface { + MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error +} + +type Move interface { + Move(ctx context.Context, srcObj, dstDir model.Obj) error +} + +type Rename interface { + Rename(ctx context.Context, srcObj model.Obj, newName string) error +} + +type Copy interface { + Copy(ctx context.Context, srcObj, dstDir model.Obj) error +} + +type Remove interface { + Remove(ctx context.Context, obj model.Obj) error +} + +type Put interface { + Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error +} + +//type WriteResult interface { +// MkdirResult +// MoveResult +// RenameResult +// CopyResult +// PutResult +// Remove +//} + +type MkdirResult interface { + MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) +} + +type MoveResult interface { + Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) +} + +type RenameResult interface { + Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) +} + +type CopyResult interface { + Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) +} + +type PutResult interface { + Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) +} + +type UpdateProgress func(percentage int) + +type Progress struct { + Total int64 + Done int64 + up UpdateProgress +} + +func (p *Progress) Write(b []byte) (n int, err error) { + n = len(b) + p.Done += int64(n) + p.up(int(float64(p.Done) / float64(p.Total) * 100)) + return +} + +func NewProgress(total int64, up UpdateProgress) *Progress { + return &Progress{ + Total: total, + up: up, + } +} diff --git a/internal/driver/item.go b/internal/driver/item.go new file mode 100644 index 000000000..6dff49d6a --- /dev/null +++ b/internal/driver/item.go @@ -0,0 +1,56 @@ +/* + * @Author: a624669980@163.com a624669980@163.com + * @Date: 2022-12-13 11:05:47 + * @LastEditors: a624669980@163.com a624669980@163.com + * @LastEditTime: 2022-12-13 11:05:54 + * @FilePath: /drive/internal/driver/item.go + * @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + */ +package driver + +type Additional interface{} + +type Select string + +type Item struct { + Name string `json:"name"` + Type string `json:"type"` + Default string `json:"default"` + Options string `json:"options"` + Required bool `json:"required"` + Help string `json:"help"` +} + +type Info struct { + Common []Item `json:"common"` + Additional []Item `json:"additional"` + Config Config `json:"config"` +} + +type IRootPath interface { + GetRootPath() string +} + +type IRootId interface { + GetRootId() string +} + +type RootPath struct { + RootFolderPath string `json:"root_folder_path"` +} + +type RootID struct { + RootFolderID string `json:"root_folder_id" omit:"true"` +} + +func (r RootPath) GetRootPath() string { + return r.RootFolderPath +} + +func (r *RootPath) SetRootPath(path string) { + r.RootFolderPath = path +} + +func (r RootID) GetRootId() string { + return r.RootFolderID +} diff --git a/internal/op/const.go b/internal/op/const.go new file mode 100644 index 000000000..a465735f9 --- /dev/null +++ b/internal/op/const.go @@ -0,0 +1,6 @@ +package op + +const ( + WORK = "work" + RootName = "root" +) diff --git a/internal/op/driver.go b/internal/op/driver.go new file mode 100644 index 000000000..ba7564118 --- /dev/null +++ b/internal/op/driver.go @@ -0,0 +1,173 @@ +package op + +import ( + "reflect" + "strings" + + "github.com/IceWhaleTech/CasaOS/internal/conf" + + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/pkg/errors" +) + +type New func() driver.Driver + +var driverNewMap = map[string]New{} +var driverInfoMap = map[string][]driver.Item{} //driver.Info{} + +func RegisterDriver(driver New) { + // log.Infof("register driver: [%s]", config.Name) + tempDriver := driver() + tempConfig := tempDriver.Config() + registerDriverItems(tempConfig, tempDriver.GetAddition()) + driverNewMap[tempConfig.Name] = driver +} + +func GetDriverNew(name string) (New, error) { + n, ok := driverNewMap[name] + if !ok { + return nil, errors.Errorf("no driver named: %s", name) + } + return n, nil +} + +func GetDriverNames() []string { + var driverNames []string + for k := range driverInfoMap { + driverNames = append(driverNames, k) + } + return driverNames +} + +// func GetDriverInfoMap() map[string]driver.Info { +// return driverInfoMap +// } +func GetDriverInfoMap() map[string][]driver.Item { + return driverInfoMap +} +func registerDriverItems(config driver.Config, addition driver.Additional) { + // log.Debugf("addition of %s: %+v", config.Name, addition) + tAddition := reflect.TypeOf(addition) + for tAddition.Kind() == reflect.Pointer { + tAddition = tAddition.Elem() + } + //mainItems := getMainItems(config) + additionalItems := getAdditionalItems(tAddition, config.DefaultRoot) + driverInfoMap[config.Name] = additionalItems + // driver.Info{ + // Common: mainItems, + // Additional: additionalItems, + // Config: config, + // } +} + +func getMainItems(config driver.Config) []driver.Item { + items := []driver.Item{{ + Name: "mount_path", + Type: conf.TypeString, + Required: true, + Help: "", + }, { + Name: "order", + Type: conf.TypeNumber, + Help: "use to sort", + }, { + Name: "remark", + Type: conf.TypeText, + }} + if !config.NoCache { + items = append(items, driver.Item{ + Name: "cache_expiration", + Type: conf.TypeNumber, + Default: "30", + Required: true, + Help: "The cache expiration time for this storage", + }) + } + if !config.OnlyProxy && !config.OnlyLocal { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "302_redirect", + Required: true, + }, + }...) + } else { + items = append(items, driver.Item{ + Name: "webdav_policy", + Type: conf.TypeSelect, + Default: "native_proxy", + Options: "use_proxy_url,native_proxy", + Required: true, + }) + } + items = append(items, driver.Item{ + Name: "down_proxy_url", + Type: conf.TypeText, + }) + if config.LocalSort { + items = append(items, []driver.Item{{ + Name: "order_by", + Type: conf.TypeSelect, + Options: "name,size,modified", + }, { + Name: "order_direction", + Type: conf.TypeSelect, + Options: "asc,desc", + }}...) + } + items = append(items, driver.Item{ + Name: "extract_folder", + Type: conf.TypeSelect, + Options: "front,back", + }) + return items +} + +func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { + var items []driver.Item + for i := 0; i < t.NumField(); i++ { + + field := t.Field(i) + if field.Type.Kind() == reflect.Struct { + items = append(items, getAdditionalItems(field.Type, defaultRoot)...) + continue + } + tag := field.Tag + ignore, ok1 := tag.Lookup("ignore") + name, ok2 := tag.Lookup("json") + if (ok1 && ignore == "true") || !ok2 { + continue + } + if tag.Get("omit") == "true" { + continue + } + item := driver.Item{ + Name: name, + Type: strings.ToLower(field.Type.Name()), + Default: tag.Get("default"), + Options: tag.Get("options"), + Required: tag.Get("required") == "true", + Help: tag.Get("help"), + } + if tag.Get("type") != "" { + item.Type = tag.Get("type") + } + if item.Name == "root_folder_id" || item.Name == "root_folder_path" { + if item.Default == "" { + item.Default = defaultRoot + } + item.Required = item.Default != "" + } + // set default type to string + if item.Type == "" { + item.Type = "string" + } + items = append(items, item) + } + return items +} diff --git a/internal/op/fs.go b/internal/op/fs.go new file mode 100644 index 000000000..576a4b664 --- /dev/null +++ b/internal/op/fs.go @@ -0,0 +1,545 @@ +package op + +import ( + "context" + "os" + stdpath "path" + "time" + + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/generic_sync" + "github.com/IceWhaleTech/CasaOS/pkg/singleflight" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/Xhofe/go-cache" + "github.com/pkg/errors" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// In order to facilitate adding some other things before and after file op + +var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) +var listG singleflight.Group[[]model.Obj] + +func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, obj := range objs { + if obj.GetName() == oldObj.GetName() { + objs[i] = newObj + break + } + } + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +func delCacheObj(storage driver.Driver, path string, obj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, oldObj := range objs { + if oldObj.GetName() == obj.GetName() { + objs = append(objs[:i], objs[i+1:]...) + break + } + } + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +var addSortDebounceMap generic_sync.MapOf[string, func(func())] + +func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, obj := range objs { + if obj.GetName() == newObj.GetName() { + objs[i] = newObj + return + } + } + + // Simple separation of files and folders + if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { + objs = append(objs, newObj) + } else { + objs = append([]model.Obj{newObj}, objs...) + } + + if storage.Config().LocalSort { + debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) + log.Debug("addCacheObj: wait start sort") + debounce(func() { + log.Debug("addCacheObj: start sort") + model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + addSortDebounceMap.Delete(key) + }) + } + + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +func ClearCache(storage driver.Driver, path string) { + listCache.Del(Key(storage, path)) +} + +func Key(storage driver.Driver, path string) string { + return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) +} + +// List files in storage, not contains virtual file +func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + log.Debugf("op.List %s", path) + key := Key(storage, path) + if !utils.IsBool(refresh...) { + if files, ok := listCache.Get(key); ok { + log.Debugf("use cache when list %s", path) + return files, nil + } + } + dir, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } + log.Debugf("list dir: %+v", dir) + if !dir.IsDir() { + return nil, errors.WithStack(errors.New("not a folder")) + } + objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + files, err := storage.List(ctx, dir, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to list objs") + } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { + s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) + // call hooks + go func(reqPath string, files []model.Obj) { + for _, hook := range ObjsUpdateHooks { + hook(args.ReqPath, files) + } + }(args.ReqPath, files) + + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + + if !storage.Config().NoCache { + if len(files) > 0 { + log.Debugf("set cache: %s => %+v", key, files) + listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } else { + log.Debugf("del cache: %s", key) + listCache.Del(key) + } + } + return files, nil + }) + return objs, err +} + +// Get object from list of files +func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + path = utils.FixAndCleanPath(path) + log.Debugf("op.Get %s", path) + + // is root folder + if utils.PathEqual(path, "/") { + var rootObj model.Obj + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + Path: path, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + default: + if storage, ok := storage.(driver.Getter); ok { + obj, err := storage.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + rootObj = obj + } + } + if rootObj == nil { + return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method") + } + return &model.ObjWrapName{ + Name: RootName, + Obj: rootObj, + }, nil + } + + // not root folder + dir, name := stdpath.Split(path) + files, err := List(ctx, storage, dir, model.ListArgs{}) + if err != nil { + return nil, errors.WithMessage(err, "failed get parent list") + } + for _, f := range files { + // TODO maybe copy obj here + if f.GetName() == name { + return f, nil + } + } + log.Debugf("cant find obj with name: %s", name) + return nil, errors.WithStack(errors.New("object not found")) +} + +func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + obj, err := Get(ctx, storage, path) + if err != nil { + return nil, err + } + return model.UnwrapObjs(obj), err +} + +var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) +var linkG singleflight.Group[*model.Link] + +// Link get link, if is an url. should have an expiry time +func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + file, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if file.IsDir() { + return nil, nil, errors.WithStack(errors.New("not a file")) + } + key := Key(storage, path) + ":" + args.IP + if link, ok := linkCache.Get(key); ok { + return link, file, nil + } + fn := func() (*model.Link, error) { + link, err := storage.Link(ctx, file, args) + if err != nil { + return nil, errors.Wrapf(err, "failed get link") + } + if link.Expiration != nil { + linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration)) + } + return link, nil + } + link, err, _ := linkG.Do(key, fn) + return link, file, err +} + +// Other api +func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { + obj, err := GetUnwrap(ctx, storage, args.Path) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get obj") + } + if o, ok := storage.(driver.Other); ok { + return o.Other(ctx, model.OtherArgs{ + Obj: obj, + Method: args.Method, + Data: args.Data, + }) + } else { + return nil, errors.New("not implement") + } +} + +var mkdirG singleflight.Group[interface{}] + +func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + key := Key(storage, path) + _, err, _ := mkdirG.Do(key, func() (interface{}, error) { + // check if dir exists + f, err := GetUnwrap(ctx, storage, path) + if err != nil { + if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { + parentPath, dirName := stdpath.Split(path) + err = MakeDir(ctx, storage, parentPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) + } + parentDir, err := GetUnwrap(ctx, storage, parentPath) + // this should not happen + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } + + switch s := storage.(type) { + case driver.MkdirResult: + var newObj model.Obj + newObj, err = s.MakeDir(ctx, parentDir, dirName) + if err == nil { + if newObj != nil { + addCacheObj(storage, parentPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, parentPath) + } + } + case driver.Mkdir: + err = s.MakeDir(ctx, parentDir, dirName) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, parentPath) + } + default: + return nil, errors.New("not implement") + } + return nil, errors.WithStack(err) + } + return nil, errors.WithMessage(err, "failed to check if dir exists") + } + // dir exists + if f.IsDir() { + return nil, nil + } + // dir to make is a file + return nil, errors.New("file exists") + }) + return err +} + +func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcRawObj, err := Get(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + srcObj := model.UnwrapObjs(srcRawObj) + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + srcDirPath := stdpath.Dir(srcPath) + + switch s := storage.(type) { + case driver.MoveResult: + var newObj model.Obj + newObj, err = s.Move(ctx, srcObj, dstDir) + if err == nil { + delCacheObj(storage, srcDirPath, srcRawObj) + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Move: + err = s.Move(ctx, srcObj, dstDir) + if err == nil { + delCacheObj(storage, srcDirPath, srcRawObj) + if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + srcRawObj, err := Get(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + srcObj := model.UnwrapObjs(srcRawObj) + srcDirPath := stdpath.Dir(srcPath) + + switch s := storage.(type) { + case driver.RenameResult: + var newObj model.Obj + newObj, err = s.Rename(ctx, srcObj, dstName) + if err == nil { + if newObj != nil { + updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, srcDirPath) + } + } + case driver.Rename: + err = s.Rename(ctx, srcObj, dstName) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, srcDirPath) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +// Copy Just copy file[s] in a storage +func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + + switch s := storage.(type) { + case driver.CopyResult: + var newObj model.Obj + newObj, err = s.Copy(ctx, srcObj, dstDir) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Copy: + err = s.Copy(ctx, srcObj, dstDir) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Remove(ctx context.Context, storage driver.Driver, path string) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + rawObj, err := Get(ctx, storage, path) + if err != nil { + // if object not found, it's ok + if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { + return nil + } + return errors.WithMessage(err, "failed to get object") + } + dirPath := stdpath.Dir(path) + + switch s := storage.(type) { + case driver.Remove: + err = s.Remove(ctx, model.UnwrapObjs(rawObj)) + if err == nil { + delCacheObj(storage, dirPath, rawObj) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + defer func() { + if f, ok := file.GetReadCloser().(*os.File); ok { + err := os.RemoveAll(f.Name()) + if err != nil { + log.Errorf("failed to remove file [%s]", f.Name()) + } + } + }() + defer func() { + if err := file.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + // if file exist and size = 0, delete it + dstDirPath = utils.FixAndCleanPath(dstDirPath) + dstPath := stdpath.Join(dstDirPath, file.GetName()) + fi, err := GetUnwrap(ctx, storage, dstPath) + if err == nil { + if fi.GetSize() == 0 { + err = Remove(ctx, storage, dstPath) + if err != nil { + return errors.WithMessagef(err, "failed remove file that exist and have size 0") + } + } else { + file.Old = fi + } + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) + } + parentDir, err := GetUnwrap(ctx, storage, dstDirPath) + // this should not happen + if err != nil { + return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + // if up is nil, set a default to prevent panic + if up == nil { + up = func(p int) {} + } + + switch s := storage.(type) { + case driver.PutResult: + var newObj model.Obj + newObj, err = s.Put(ctx, parentDir, file, up) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Put: + err = s.Put(ctx, parentDir, file, up) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errors.New("not implement") + } + log.Debugf("put file [%s] done", file.GetName()) + //if err == nil { + // //clear cache + // key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath) + // listCache.Del(key) + //} + return errors.WithStack(err) +} diff --git a/internal/op/hook.go b/internal/op/hook.go new file mode 100644 index 000000000..5c07fc43f --- /dev/null +++ b/internal/op/hook.go @@ -0,0 +1,109 @@ +package op + +import ( + "regexp" + "strings" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/conf" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +// Obj +type ObjsUpdateHook = func(parent string, objs []model.Obj) + +var ( + ObjsUpdateHooks = make([]ObjsUpdateHook, 0) +) + +func RegisterObjsUpdateHook(hook ObjsUpdateHook) { + ObjsUpdateHooks = append(ObjsUpdateHooks, hook) +} + +func HandleObjsUpdateHook(parent string, objs []model.Obj) { + for _, hook := range ObjsUpdateHooks { + hook(parent, objs) + } +} + +// Setting +type SettingItemHook func(item *model.SettingItem) error + +var settingItemHooks = map[string]SettingItemHook{ + conf.VideoTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.VideoTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.AudioTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.AudioTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ImageTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ImageTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.TextTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.TextTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ProxyTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ProxyTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ProxyIgnoreHeaders: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = strings.Split(item.Value, ",") + return nil + }, + conf.PrivacyRegs: func(item *model.SettingItem) error { + regStrs := strings.Split(item.Value, "\n") + regs := make([]*regexp.Regexp, 0, len(regStrs)) + for _, regStr := range regStrs { + reg, err := regexp.Compile(regStr) + if err != nil { + return errors.WithStack(err) + } + regs = append(regs, reg) + } + conf.PrivacyReg = regs + return nil + }, + conf.FilenameCharMapping: func(item *model.SettingItem) error { + var json = jsoniter.ConfigCompatibleWithStandardLibrary + err := json.UnmarshalFromString(item.Value, &conf.FilenameCharMap) + if err != nil { + return err + } + logger.Info("filename char mapping", zap.Any("FilenameCharMap", conf.FilenameCharMap)) + return nil + }, +} + +func RegisterSettingItemHook(key string, hook SettingItemHook) { + settingItemHooks[key] = hook +} + +func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { + if hook, ok := settingItemHooks[item.Key]; ok { + return true, hook(item) + } + return false, nil +} + +// Storage +type StorageHook func(typ string, storage driver.Driver) + +var storageHooks = make([]StorageHook, 0) + +func CallStorageHooks(typ string, storage driver.Driver) { + for _, hook := range storageHooks { + hook(typ, storage) + } +} + +func RegisterStorageHook(hook StorageHook) { + storageHooks = append(storageHooks, hook) +} diff --git a/internal/sign/sign.go b/internal/sign/sign.go new file mode 100644 index 000000000..412bdfe06 --- /dev/null +++ b/internal/sign/sign.go @@ -0,0 +1,36 @@ +package sign + +import ( + "sync" + "time" + + "github.com/IceWhaleTech/CasaOS/pkg/sign" +) + +var once sync.Once +var instance sign.Sign + +func Sign(data string) string { + + return NotExpired(data) + +} + +func WithDuration(data string, d time.Duration) string { + once.Do(Instance) + return instance.Sign(data, time.Now().Add(d).Unix()) +} + +func NotExpired(data string) string { + once.Do(Instance) + return instance.Sign(data, 0) +} + +func Verify(data string, sign string) error { + once.Do(Instance) + return instance.Verify(data, sign) +} + +func Instance() { + instance = sign.NewHMACSign([]byte("token")) +} diff --git a/main.go b/main.go index b99161a41..abb564271 100644 --- a/main.go +++ b/main.go @@ -80,6 +80,11 @@ func init() { service.GetCPUThermalZone() route.InitFunction() + + /// + // service.MountLists = make(map[string]*mountlib.MountPoint) + // configfile.Install() + service.MyService.Storage().CheckAndMountAll() } // @title casaOS API @@ -141,9 +146,9 @@ func main() { "/v1/image", "/v1/samba", "/v1/notify", - //"/v1/driver", - //"/v1/cloud", - //"/v1/recover", + "/v1/driver", + "/v1/cloud", + "/v1/recover", "/v1/other", route.V2APIPath, route.V2DocPath, @@ -162,6 +167,7 @@ func main() { } var events []message_bus.EventType events = append(events, message_bus.EventType{Name: "casaos:system:utilization", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}}) + events = append(events, message_bus.EventType{Name: "casaos:file:recover", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}}) events = append(events, message_bus.EventType{Name: "casaos:file:operate", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}}) // register at message bus for i := 0; i < 10; i++ { diff --git a/model/storage.go b/model/storage.go index 2be5fa33f..e8506aa20 100644 --- a/model/storage.go +++ b/model/storage.go @@ -2,7 +2,7 @@ package model import "time" -type Storage struct { +type StorageA struct { ID uint `json:"id" gorm:"primaryKey"` // unique key MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized Order int `json:"order"` // use to sort @@ -29,15 +29,15 @@ type Proxy struct { DownProxyUrl string `json:"down_proxy_url"` } -func (s *Storage) GetStorage() *Storage { +func (s *StorageA) GetStorage() *StorageA { return s } -func (s *Storage) SetStorage(storage Storage) { +func (s *StorageA) SetStorage(storage StorageA) { *s = storage } -func (s *Storage) SetStatus(status string) { +func (s *StorageA) SetStatus(status string) { s.Status = status } diff --git a/pkg/utils/httper/drive.go b/pkg/utils/httper/drive.go index b9200de0d..80817bc1e 100644 --- a/pkg/utils/httper/drive.go +++ b/pkg/utils/httper/drive.go @@ -1,7 +1,15 @@ package httper import ( + "encoding/json" + "fmt" + "net" + "net/http" "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" ) type MountList struct { @@ -36,126 +44,127 @@ type RemotesResult struct { var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" var DefaultTimeout = time.Second * 30 -// func NewRestyClient() *resty.Client { - -// unixSocket := "/var/run/rclone/rclone.sock" - -// transport := http.Transport{ -// Dial: func(_, _ string) (net.Conn, error) { -// return net.Dial("unix", unixSocket) -// }, -// } - -// client := resty.New() - -// client.SetTransport(&transport).SetBaseURL("http://localhost") -// client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent) -// return client -// } - -// func GetMountList() (MountList, error) { -// var result MountList -// res, err := NewRestyClient().R().Post("/mount/listmounts") -// if err != nil { -// return result, err -// } -// if res.StatusCode() != 200 { -// return result, fmt.Errorf("get mount list failed") -// } -// json.Unmarshal(res.Body(), &result) -// for i := 0; i < len(result.MountPoints); i++ { -// result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1] -// } -// return result, err -// } -// -// func Mount(mountPoint string, fs string) error { -// res, err := NewRestyClient().R().SetFormData(map[string]string{ -// "mountPoint": mountPoint, -// "fs": fs, -// "mountOpt": `{"AllowOther": true}`, -// }).Post("/mount/mount") -// if err != nil { -// return err -// } -// if res.StatusCode() != 200 { -// return fmt.Errorf("mount failed") -// } -// logger.Info("mount then", zap.Any("res", res.Body())) -// return nil -// } -// func Unmount(mountPoint string) error { -// res, err := NewRestyClient().R().SetFormData(map[string]string{ -// "mountPoint": mountPoint, -// }).Post("/mount/unmount") -// if err != nil { -// logger.Error("when unmount", zap.Error(err)) -// return err -// } -// if res.StatusCode() != 200 { -// logger.Error("then unmount failed", zap.Any("res", res.Body())) -// return fmt.Errorf("unmount failed") -// } -// logger.Info("unmount then", zap.Any("res", res.Body())) -// return nil -// } - -// func CreateConfig(data map[string]string, name, t string) error { -// data["config_is_local"] = "false" -// dataStr, _ := json.Marshal(data) -// res, err := NewRestyClient().R().SetFormData(map[string]string{ -// "name": name, -// "parameters": string(dataStr), -// "type": t, -// }).Post("/config/create") -// logger.Info("when create config then", zap.Any("res", res.Body())) -// if err != nil { -// return err -// } -// if res.StatusCode() != 200 { -// return fmt.Errorf("create config failed") -// } - -// return nil -// } - -// func GetConfigByName(name string) (map[string]string, error) { - -// res, err := NewRestyClient().R().SetFormData(map[string]string{ -// "name": name, -// }).Post("/config/get") -// if err != nil { -// return nil, err -// } -// if res.StatusCode() != 200 { -// return nil, fmt.Errorf("create config failed") -// } -// var result map[string]string -// json.Unmarshal(res.Body(), &result) -// return result, nil -// } -// func GetAllConfigName() (RemotesResult, error) { -// var result RemotesResult -// res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes") -// if err != nil { -// return result, err -// } -// if res.StatusCode() != 200 { -// return result, fmt.Errorf("get config failed") -// } - -// json.Unmarshal(res.Body(), &result) -// return result, nil -// } -// func DeleteConfigByName(name string) error { -// res, err := NewRestyClient().R().SetFormData(map[string]string{ -// "name": name, -// }).Post("/config/delete") -// if err != nil { -// return err -// } -// if res.StatusCode() != 200 { -// return fmt.Errorf("delete config failed") -// } -// return nil -// } +func NewRestyClient() *resty.Client { + + unixSocket := "/var/run/rclone/rclone.sock" + + transport := http.Transport{ + Dial: func(_, _ string) (net.Conn, error) { + return net.Dial("unix", unixSocket) + }, + } + + client := resty.New() + + client.SetTransport(&transport).SetBaseURL("http://localhost") + client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent) + return client +} + +func GetMountList() (MountList, error) { + var result MountList + res, err := NewRestyClient().R().Post("/mount/listmounts") + if err != nil { + return result, err + } + if res.StatusCode() != 200 { + return result, fmt.Errorf("get mount list failed") + } + json.Unmarshal(res.Body(), &result) + for i := 0; i < len(result.MountPoints); i++ { + result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1] + } + return result, err +} + +func Mount(mountPoint string, fs string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "mountPoint": mountPoint, + "fs": fs, + "mountOpt": `{"AllowOther": true}`, + }).Post("/mount/mount") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("mount failed") + } + logger.Info("mount then", zap.Any("res", res.Body())) + return nil +} +func Unmount(mountPoint string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "mountPoint": mountPoint, + }).Post("/mount/unmount") + if err != nil { + logger.Error("when unmount", zap.Error(err)) + return err + } + if res.StatusCode() != 200 { + logger.Error("then unmount failed", zap.Any("res", res.Body())) + return fmt.Errorf("unmount failed") + } + + logger.Info("unmount then", zap.Any("res", res.Body())) + return nil +} + +func CreateConfig(data map[string]string, name, t string) error { + data["config_is_local"] = "false" + dataStr, _ := json.Marshal(data) + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + "parameters": string(dataStr), + "type": t, + }).Post("/config/create") + logger.Info("when create config then", zap.Any("res", res.Body())) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("create config failed") + } + + return nil +} + +func GetConfigByName(name string) (map[string]string, error) { + + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + }).Post("/config/get") + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, fmt.Errorf("create config failed") + } + var result map[string]string + json.Unmarshal(res.Body(), &result) + return result, nil +} +func GetAllConfigName() (RemotesResult, error) { + var result RemotesResult + res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes") + if err != nil { + return result, err + } + if res.StatusCode() != 200 { + return result, fmt.Errorf("get config failed") + } + + json.Unmarshal(res.Body(), &result) + return result, nil +} +func DeleteConfigByName(name string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + }).Post("/config/delete") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("delete config failed") + } + return nil +} diff --git a/route/v1.go b/route/v1.go index 4f20ee510..becf64320 100644 --- a/route/v1.go +++ b/route/v1.go @@ -36,7 +36,7 @@ func InitV1Router() *gin.Engine { r.GET("/ping", func(ctx *gin.Context) { ctx.String(200, "pong") }) - + r.GET("/v1/recover/:type", v1.GetRecoverStorage) v1Group := r.Group("/v1") v1Group.Use(jwt.ExceptLocalhost()) @@ -98,6 +98,17 @@ func InitV1Router() *gin.Engine { v1FileGroup.GET("/ws", v1.ConnectWebSocket) v1FileGroup.GET("/peers", v1.GetPeers) } + v1CloudGroup := v1Group.Group("/cloud") + v1CloudGroup.Use() + { + v1CloudGroup.GET("", v1.ListStorages) + v1CloudGroup.DELETE("", v1.UmountStorage) + } + v1DriverGroup := v1Group.Group("/driver") + v1DriverGroup.Use() + { + v1DriverGroup.GET("", v1.ListDriverInfo) + } v1FolderGroup := v1Group.Group("/folder") v1FolderGroup.Use() diff --git a/route/v1/cloud.go b/route/v1/cloud.go new file mode 100644 index 000000000..f4ab07b1a --- /dev/null +++ b/route/v1/cloud.go @@ -0,0 +1,101 @@ +package v1 + +import ( + "strings" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/dropbox" + "github.com/IceWhaleTech/CasaOS/drivers/google_drive" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" + "github.com/IceWhaleTech/CasaOS/pkg/utils/httper" + "github.com/IceWhaleTech/CasaOS/service" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +func ListStorages(c *gin.Context) { + // var req model.PageReq + // if err := c.ShouldBind(&req); err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + // return + // } + // req.Validate() + + //logger.Info("ListStorages", zap.Any("req", req)) + //storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + // return + // } + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{ + // Content: storages, + // Total: total, + // }}) + r, err := service.MyService.Storage().GetStorages() + + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + + for i := 0; i < len(r.MountPoints); i++ { + dataMap, err := service.MyService.Storage().GetConfigByName(r.MountPoints[i].Fs) + if err != nil { + logger.Error("GetConfigByName", zap.Any("err", err)) + continue + } + if dataMap["type"] == "drive" { + r.MountPoints[i].Icon = google_drive.ICONURL + } + if dataMap["type"] == "dropbox" { + r.MountPoints[i].Icon = dropbox.ICONURL + } + r.MountPoints[i].Name = dataMap["username"] + } + list := []httper.MountPoint{} + + for _, v := range r.MountPoints { + list = append(list, httper.MountPoint{ + Fs: v.Fs, + Icon: v.Icon, + MountPoint: v.MountPoint, + Name: v.Name, + }) + } + + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: list}) +} + +func UmountStorage(c *gin.Context) { + json := make(map[string]string) + c.ShouldBind(&json) + mountPoint := json["mount_point"] + if mountPoint == "" { + c.JSON(common_err.CLIENT_ERROR, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: "mount_point is empty"}) + return + } + err := service.MyService.Storage().UnmountStorage(mountPoint) + if err != nil { + c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + service.MyService.Storage().DeleteConfigByName(strings.ReplaceAll(mountPoint, "/mnt/", "")) + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) +} + +func GetStorage(c *gin.Context) { + + // idStr := c.Query("id") + // id, err := strconv.Atoi(idStr) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + // return + // } + // storage, err := service.MyService.Storage().GetStorageById(uint(id)) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + // return + // } + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage}) +} diff --git a/route/v1/driver.go b/route/v1/driver.go new file mode 100644 index 000000000..88c96d81e --- /dev/null +++ b/route/v1/driver.go @@ -0,0 +1,12 @@ +package v1 + +import ( + "github.com/IceWhaleTech/CasaOS-Common/model" + "github.com/IceWhaleTech/CasaOS-Common/utils/common_err" + "github.com/IceWhaleTech/CasaOS/internal/op" + "github.com/gin-gonic/gin" +) + +func ListDriverInfo(c *gin.Context) { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: op.GetDriverInfoMap()}) +} diff --git a/route/v1/recover.go b/route/v1/recover.go new file mode 100644 index 000000000..e2f554ef2 --- /dev/null +++ b/route/v1/recover.go @@ -0,0 +1,205 @@ +package v1 + +import ( + "strconv" + "strings" + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/dropbox" + "github.com/IceWhaleTech/CasaOS/drivers/google_drive" + "github.com/IceWhaleTech/CasaOS/service" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +func GetRecoverStorage(c *gin.Context) { + c.Header("Content-Type", "text/html; charset=utf-8") + t := c.Param("type") + currentTime := time.Now().UTC() + currentDate := time.Now().UTC().Format("2006-01-02") + notify := make(map[string]interface{}) + if t == "GoogleDrive" { + add := google_drive.Addition{} + add.Code = c.Query("code") + if len(add.Code) == 0 { + c.String(200, `

Code cannot be empty

`) + notify["status"] = "fail" + notify["message"] = "Code cannot be empty" + logger.Error("Then code is empty: ", zap.String("code", add.Code), zap.Any("name", "google_drive")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + + add.RootFolderID = "root" + add.ClientID = google_drive.CLIENTID + add.ClientSecret = google_drive.CLIENTSECRET + + var google_drive google_drive.GoogleDrive + google_drive.Addition = add + err := google_drive.Init(c) + if err != nil { + c.String(200, `

Initialization failure:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Initialization failure" + logger.Error("Then init error: ", zap.Error(err), zap.Any("name", "google_drive")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + + username, err := google_drive.GetUserInfo(c) + if err != nil { + c.String(200, `

Failed to get user information:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Failed to get user information" + logger.Error("Then get user info error: ", zap.Error(err), zap.Any("name", "google_drive")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + dmap := make(map[string]string) + dmap["username"] = username + configs, err := service.MyService.Storage().GetConfig() + if err != nil { + c.String(200, `

Failed to get rclone config:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Failed to get rclone config" + logger.Error("Then get config error: ", zap.Error(err), zap.Any("name", "google_drive")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + for _, v := range configs.Remotes { + cf, err := service.MyService.Storage().GetConfigByName(v) + if err != nil { + logger.Error("then get config by name error: ", zap.Error(err), zap.Any("name", v)) + continue + } + if cf["type"] == "drive" && cf["username"] == dmap["username"] { + c.String(200, `

The same configuration has been added

`) + err := service.MyService.Storage().CheckAndMountByName(v) + if err != nil { + logger.Error("check and mount by name error: ", zap.Error(err), zap.Any("name", cf["username"])) + } + notify["status"] = "warn" + notify["message"] = "The same configuration has been added" + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + } + if len(username) > 0 { + a := strings.Split(username, "@") + username = a[0] + } + + //username = fileutil.NameAccumulation(username, "/mnt") + username += "_google_drive_" + strconv.FormatInt(time.Now().Unix(), 10) + + dmap["client_id"] = add.ClientID + dmap["client_secret"] = add.ClientSecret + dmap["scope"] = "drive" + dmap["mount_point"] = "/mnt/" + username + dmap["token"] = `{"access_token":"` + google_drive.AccessToken + `","token_type":"Bearer","refresh_token":"` + google_drive.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*1).Add(time.Minute*50).Format("15:04:05") + `Z"}` + service.MyService.Storage().CreateConfig(dmap, username, "drive") + service.MyService.Storage().MountStorage("/mnt/"+username, username+":") + notify := make(map[string]interface{}) + notify["status"] = "success" + notify["message"] = "Success" + notify["driver"] = "GoogleDrive" + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + } else if t == "Dropbox" { + add := dropbox.Addition{} + add.Code = c.Query("code") + if len(add.Code) == 0 { + c.String(200, `

Code cannot be empty

`) + notify["status"] = "fail" + notify["message"] = "Code cannot be empty" + logger.Error("Then code is empty error: ", zap.String("code", add.Code), zap.Any("name", "dropbox")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + add.RootFolderID = "" + add.AppKey = dropbox.APPKEY + add.AppSecret = dropbox.APPSECRET + var dropbox dropbox.Dropbox + dropbox.Addition = add + err := dropbox.Init(c) + if err != nil { + c.String(200, `

Initialization failure:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Initialization failure" + logger.Error("Then init error: ", zap.Error(err), zap.Any("name", "dropbox")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + username, err := dropbox.GetUserInfo(c) + if err != nil { + c.String(200, `

Failed to get user information:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Failed to get user information" + logger.Error("Then get user information: ", zap.Error(err), zap.Any("name", "dropbox")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + dmap := make(map[string]string) + dmap["username"] = username + + configs, err := service.MyService.Storage().GetConfig() + if err != nil { + c.String(200, `

Failed to get rclone config:`+err.Error()+`

`) + notify["status"] = "fail" + notify["message"] = "Failed to get rclone config" + logger.Error("Then get config error: ", zap.Error(err), zap.Any("name", "dropbox")) + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + for _, v := range configs.Remotes { + cf, err := service.MyService.Storage().GetConfigByName(v) + if err != nil { + logger.Error("then get config by name error: ", zap.Error(err), zap.Any("name", v)) + continue + } + if cf["type"] == "dropbox" && cf["username"] == dmap["username"] { + c.String(200, `

The same configuration has been added

`) + err := service.MyService.Storage().CheckAndMountByName(v) + if err != nil { + logger.Error("check and mount by name error: ", zap.Error(err), zap.Any("name", cf["username"])) + } + + notify["status"] = "warn" + notify["message"] = "The same configuration has been added" + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + return + } + } + if len(username) > 0 { + a := strings.Split(username, "@") + username = a[0] + } + username += "_dropbox_" + strconv.FormatInt(time.Now().Unix(), 10) + + dmap["client_id"] = add.AppKey + dmap["client_secret"] = add.AppSecret + dmap["token"] = `{"access_token":"` + dropbox.AccessToken + `","token_type":"bearer","refresh_token":"` + dropbox.Addition.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*3).Add(time.Minute*50).Format("15:04:05") + `.780385354Z"}` + dmap["mount_point"] = "/mnt/" + username + // data.SetValue(username, "type", "dropbox") + // data.SetValue(username, "client_id", add.AppKey) + // data.SetValue(username, "client_secret", add.AppSecret) + // data.SetValue(username, "mount_point", "/mnt/"+username) + + // data.SetValue(username, "token", `{"access_token":"`+dropbox.AccessToken+`","token_type":"bearer","refresh_token":"`+dropbox.Addition.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*3).Format("15:04:05")+`.780385354Z"}`) + // e = data.Save() + // if e != nil { + // c.String(200, `

保存配置失败:`+e.Error()+`

`) + + // return + // } + service.MyService.Storage().CreateConfig(dmap, username, "dropbox") + service.MyService.Storage().MountStorage("/mnt/"+username, username+":") + + notify["status"] = "success" + notify["message"] = "Success" + notify["driver"] = "Dropbox" + service.MyService.Notify().SendNotify("casaos:file:recover", notify) + } + + c.String(200, `

Just close the page

`) +} diff --git a/service/service.go b/service/service.go index 1bb9171f0..3c4e57fc3 100644 --- a/service/service.go +++ b/service/service.go @@ -39,7 +39,7 @@ type Repository interface { Rely() RelyService Shares() SharesService System() SystemService - + Storage() StorageService MessageBus() *message_bus.ClientWithResponses Peer() PeerService Other() OtherService @@ -60,9 +60,10 @@ func NewService(db *gorm.DB, RuntimePath string) Repository { system: NewSystemService(), health: NewHealthService(), shares: NewSharesService(db), + storage: NewStorageService(), + other: NewOtherService(), - peer: NewPeerService(db), - other: NewOtherService(), + peer: NewPeerService(db), } } @@ -76,9 +77,13 @@ type store struct { shares SharesService connections ConnectionsService gateway external.ManagementService + storage StorageService + health HealthService + other OtherService +} - health HealthService - other OtherService +func (c *store) Storage() StorageService { + return c.storage } func (c *store) Peer() PeerService { diff --git a/service/storage.go b/service/storage.go new file mode 100644 index 000000000..a13711e43 --- /dev/null +++ b/service/storage.go @@ -0,0 +1,116 @@ +package service + +import ( + "io/ioutil" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/pkg/utils/file" + "github.com/IceWhaleTech/CasaOS/pkg/utils/httper" + "go.uber.org/zap" +) + +type StorageService interface { + MountStorage(mountPoint, fs string) error + UnmountStorage(mountPoint string) error + GetStorages() (httper.MountList, error) + CreateConfig(data map[string]string, name string, t string) error + CheckAndMountByName(name string) error + CheckAndMountAll() error + GetConfigByName(name string) (map[string]string, error) + DeleteConfigByName(name string) error + GetConfig() (httper.RemotesResult, error) +} + +type storageStruct struct { +} + +func (s *storageStruct) MountStorage(mountPoint, fs string) error { + file.IsNotExistMkDir(mountPoint) + return httper.Mount(mountPoint, fs) +} +func (s *storageStruct) UnmountStorage(mountPoint string) error { + err := httper.Unmount(mountPoint) + if err == nil { + dir, _ := ioutil.ReadDir(mountPoint) + + if len(dir) == 0 { + file.RMDir(mountPoint) + } + return nil + } + return err +} +func (s *storageStruct) GetStorages() (httper.MountList, error) { + return httper.GetMountList() +} +func (s *storageStruct) CreateConfig(data map[string]string, name string, t string) error { + httper.CreateConfig(data, name, t) + return nil +} +func (s *storageStruct) CheckAndMountByName(name string) error { + storages, _ := MyService.Storage().GetStorages() + currentRemote, _ := httper.GetConfigByName(name) + mountPoint := currentRemote["mount_point"] + isMount := false + for _, v := range storages.MountPoints { + if v.MountPoint == mountPoint { + isMount = true + break + } + } + if !isMount { + return MyService.Storage().MountStorage(mountPoint, name+":") + } + return nil +} +func (s *storageStruct) CheckAndMountAll() error { + storages, err := MyService.Storage().GetStorages() + if err != nil { + return err + } + logger.Info("when CheckAndMountAll storages", zap.Any("storages", storages)) + section, err := httper.GetAllConfigName() + if err != nil { + return err + } + logger.Info("when CheckAndMountAll section", zap.Any("section", section)) + for _, v := range section.Remotes { + currentRemote, _ := httper.GetConfigByName(v) + mountPoint := currentRemote["mount_point"] + if len(mountPoint) == 0 { + continue + } + isMount := false + for _, v := range storages.MountPoints { + if v.MountPoint == mountPoint { + isMount = true + break + } + } + if !isMount { + logger.Info("when CheckAndMountAll MountStorage", zap.String("mountPoint", mountPoint), zap.String("fs", v)) + err := MyService.Storage().MountStorage(mountPoint, v+":") + if err != nil { + logger.Error("when CheckAndMountAll then", zap.String("mountPoint", mountPoint), zap.String("fs", v), zap.Error(err)) + } + } + } + return nil +} + +func (s *storageStruct) GetConfigByName(name string) (map[string]string, error) { + return httper.GetConfigByName(name) +} +func (s *storageStruct) DeleteConfigByName(name string) error { + return httper.DeleteConfigByName(name) +} +func (s *storageStruct) GetConfig() (httper.RemotesResult, error) { + section, err := httper.GetAllConfigName() + if err != nil { + return httper.RemotesResult{}, err + } + return section, nil +} +func NewStorageService() StorageService { + return &storageStruct{} +} diff --git a/service/storage_old.go b/service/storage_old.go deleted file mode 100644 index d5a928a18..000000000 --- a/service/storage_old.go +++ /dev/null @@ -1,73 +0,0 @@ -package service - -import ( - "fmt" - - "github.com/IceWhaleTech/CasaOS/model" - "github.com/pkg/errors" - "gorm.io/gorm" -) - -type StorageOldService interface { - CreateStorage(storage *model.Storage) error - UpdateStorage(storage *model.Storage) error - DeleteStorageById(id uint) error - GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) - GetStorageById(id uint) (*model.Storage, error) - GetEnabledStorages() ([]model.Storage, error) -} - -type storageOldStruct struct { - db *gorm.DB -} - -// CreateStorage just insert storage to database -func (s *storageOldStruct) CreateStorage(storage *model.Storage) error { - return errors.WithStack(s.db.Create(storage).Error) -} - -// UpdateStorage just update storage in database -func (s *storageOldStruct) UpdateStorage(storage *model.Storage) error { - return errors.WithStack(s.db.Save(storage).Error) -} - -// DeleteStorageById just delete storage from database by id -func (s *storageOldStruct) DeleteStorageById(id uint) error { - return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error) -} - -// GetStorages Get all storages from database order by index -func (s *storageOldStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { - storageDB := s.db.Model(&model.Storage{}) - var count int64 - if err := storageDB.Count(&count).Error; err != nil { - return nil, 0, errors.Wrapf(err, "failed get storages count") - } - var storages []model.Storage - if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { - return nil, 0, errors.WithStack(err) - } - return storages, count, nil -} - -// GetStorageById Get Storage by id, used to update storage usually -func (s *storageOldStruct) GetStorageById(id uint) (*model.Storage, error) { - var storage model.Storage - storage.ID = id - if err := s.db.First(&storage).Error; err != nil { - return nil, errors.WithStack(err) - } - return &storage, nil -} - -func (s *storageOldStruct) GetEnabledStorages() ([]model.Storage, error) { - var storages []model.Storage - if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil { - return nil, errors.WithStack(err) - } - return storages, nil -} - -func NewStorageOldService(db *gorm.DB) StorageOldService { - return &storageOldStruct{db: db} -} diff --git a/service/system.go b/service/system.go index 4c29c858f..317022f35 100644 --- a/service/system.go +++ b/service/system.go @@ -169,7 +169,7 @@ func (c *systemService) GetDirPath(path string) ([]model.Path, error) { } - ls, err := ioutil.ReadDir(path) + ls, err := os.ReadDir(path) if err != nil { logger.Error("when read dir", zap.Error(err)) return []model.Path{}, err @@ -182,7 +182,12 @@ func (c *systemService) GetDirPath(path string) ([]model.Path, error) { if err != nil { link = filePath } - temp := model.Path{Name: l.Name(), Path: filePath, IsDir: l.IsDir(), Date: l.ModTime(), Size: l.Size()} + tempFile, err := l.Info() + if err != nil { + logger.Error("when read dir", zap.Error(err)) + return []model.Path{}, err + } + temp := model.Path{Name: l.Name(), Path: filePath, IsDir: l.IsDir(), Date: tempFile.ModTime(), Size: tempFile.Size()} if filePath != link { file, _ := os.Stat(link) temp.IsDir = file.IsDir()