beenull
/
CasaOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Test rclone (#961)

This commit is contained in:
link 2023-03-17 15:37:28 +08:00 committed by GitHub
parent 59f2ccbeb3
commit 10191a1be3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 2675 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
cmd/migration-tool/migration_042.go
View File

@ -1,67 +0,0 @@
/*
* @Author: LinkLeong link@icewhale.org
* @Date: 2022-08-24 17:36:00
* @LastEditors: LinkLeong
* @LastEditTime: 2022-09-05 11:24:27
* @FilePath: /CasaOS/cmd/migration-tool/migration-034-035.go
* @Description:
* @Website: https://www.casaos.io
* Copyright (c) 2022 by icewhale, All Rights Reserved.
*/
package main
import (
"os"
interfaces "github.com/IceWhaleTech/CasaOS-Common"
"github.com/IceWhaleTech/CasaOS-Common/utils/version"
"github.com/IceWhaleTech/CasaOS/pkg/utils/command"
)
type migrationTool struct{}
func (u *migrationTool) IsMigrationNeeded() (bool, error) {
majorVersion, minorVersion, patchVersion, err := version.DetectLegacyVersion()
if err != nil {
if err == version.ErrLegacyVersionNotFound {
return false, nil
}
return false, err
}
if majorVersion > 0 {
return false, nil
}
if minorVersion > 4 {
return false, nil
}
if minorVersion == 4 && patchVersion != 2 {
return false, nil
}
_logger.Info("Migration is needed for a CasaOS version 0.4.2 ")
return true, nil
}
func (u *migrationTool) PreMigrate() error {
return nil
}
func (u *migrationTool) Migrate() error {
_logger.Info("Migration is started for a CasaOS version 0.4.2 ")
command.OnlyExec("systemctl stop rclone.service")
os.Remove("/usr/lib/systemd/system/rclone.service")
command.OnlyExec("systemctl daemon-reload")
return nil
}
func (u *migrationTool) PostMigrate() error {
return nil
}
func NewMigrationDummy() interfaces.MigrationTool {
return &migrationTool{}
}

27
cmd/migration-tool/migration_dummy.go Normal file
View File

@ -0,0 +1,27 @@
package main
import (
interfaces "github.com/IceWhaleTech/CasaOS-Common"
)
type migrationTool struct{}
func (u *migrationTool) IsMigrationNeeded() (bool, error) {
return false, nil
}
func (u *migrationTool) PreMigrate() error {
return nil
}
func (u *migrationTool) Migrate() error {
return nil
}
func (u *migrationTool) PostMigrate() error {
return nil
}
func NewMigrationDummy() interfaces.MigrationTool {
return &migrationTool{}
}

12
drivers/all.go Normal file
View File

@ -0,0 +1,12 @@
package drivers
import (
_ "github.com/IceWhaleTech/CasaOS/drivers/dropbox"
_ "github.com/IceWhaleTech/CasaOS/drivers/google_drive"
)
// All do nothing,just for import
// same as _ import
func All() {
}

30
drivers/base/client.go Normal file
View File

@ -0,0 +1,30 @@
package base
import (
"net/http"
"time"
"github.com/go-resty/resty/v2"
)
var NoRedirectClient *resty.Client
var RestyClient = NewRestyClient()
var HttpClient = &http.Client{}
var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
var DefaultTimeout = time.Second * 30
func init() {
NoRedirectClient = resty.New().SetRedirectPolicy(
resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}),
)
NoRedirectClient.SetHeader("user-agent", UserAgent)
}
func NewRestyClient() *resty.Client {
return resty.New().
SetHeader("user-agent", UserAgent).
SetRetryCount(3).
SetTimeout(DefaultTimeout)
}

12
drivers/base/types.go Normal file
View File

@ -0,0 +1,12 @@
package base
import "github.com/go-resty/resty/v2"
type Json map[string]interface{}
type TokenResp struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
}
type ReqCallback func(req *resty.Request)

100
drivers/dropbox/drive.go Normal file
View File

@ -0,0 +1,100 @@
package dropbox
import (
"context"
"errors"
"net/http"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
type Dropbox struct {
model.StorageA
Addition
AccessToken string
}
func (d *Dropbox) Config() driver.Config {
return config
}
func (d *Dropbox) GetAddition() driver.Additional {
return &d.Addition
}
func (d *Dropbox) Init(ctx context.Context) error {
if len(d.RefreshToken) == 0 {
d.getRefreshToken()
}
return d.refreshToken()
}
func (d *Dropbox) Drop(ctx context.Context) error {
return nil
}
func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
files, err := d.getFiles(dir.GetID())
if err != nil {
return nil, err
}
return utils.SliceConvert(files, func(src File) (model.Obj, error) {
return fileToObj(src), nil
})
}
func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
url := "https://content.dropboxapi.com/2/files/download"
link := model.Link{
URL: url,
Method: http.MethodPost,
Header: http.Header{
"Authorization": []string{"Bearer " + d.AccessToken},
"Dropbox-API-Arg": []string{`{"path": "` + file.GetPath() + `"}`},
},
}
return &link, nil
}
func (d *Dropbox) GetUserInfo(ctx context.Context) (string, error) {
url := "https://api.dropboxapi.com/2/users/get_current_account"
user := UserInfo{}
resp, err := d.request(url, http.MethodPost, func(req *resty.Request) {
req.SetHeader("Content-Type", "")
}, &user)
if err != nil {
return "", err
}
logger.Info("resp", zap.Any("resp", string(resp)))
return user.Email, nil
}
func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
return nil
}
func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
return nil
}
func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
return nil
}
func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
return errors.New("not support")
}
func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error {
return nil
}
func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
return nil
}
var _ driver.Driver = (*Dropbox)(nil)

33
drivers/dropbox/meta.go Normal file
View File

@ -0,0 +1,33 @@
package dropbox
import (
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/internal/op"
)
const ICONURL = "./img/driver/Dropbox.svg"
const APPKEY = "tciqajyazzdygt9"
const APPSECRET = "e7gtmv441cwdf0n"
type Addition struct {
driver.RootID
RefreshToken string `json:"refresh_token" required:"true" omit:"true"`
AppKey string `json:"app_key" type:"string" default:"tciqajyazzdygt9" omit:"true"`
AppSecret string `json:"app_secret" type:"string" default:"e7gtmv441cwdf0n" omit:"true"`
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`
AuthUrl string `json:"auth_url" type:"string" default:"https://www.dropbox.com/oauth2/authorize?client_id=tciqajyazzdygt9&redirect_uri=https://cloudoauth.files.casaos.app&response_type=code&token_access_type=offline&state=${HOST}%2Fv1%2Frecover%2FDropbox&&force_reapprove=true&force_reauthentication=true"`
Icon string `json:"icon" type:"string" default:"./img/driver/Dropbox.svg"`
Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`
}
var config = driver.Config{
Name: "Dropbox",
OnlyProxy: true,
DefaultRoot: "root",
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &Dropbox{}
})
}

88
drivers/dropbox/types.go Normal file
View File

@ -0,0 +1,88 @@
package dropbox
import (
"time"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/model"
"go.uber.org/zap"
)
type UserInfo struct {
AccountID string `json:"account_id"`
Name struct {
GivenName string `json:"given_name"`
Surname string `json:"surname"`
FamiliarName string `json:"familiar_name"`
DisplayName string `json:"display_name"`
AbbreviatedName string `json:"abbreviated_name"`
} `json:"name"`
Email string `json:"email"`
EmailVerified bool `json:"email_verified"`
Disabled bool `json:"disabled"`
Country string `json:"country"`
Locale string `json:"locale"`
ReferralLink string `json:"referral_link"`
IsPaired bool `json:"is_paired"`
AccountType struct {
Tag string `json:".tag"`
} `json:"account_type"`
RootInfo struct {
Tag string `json:".tag"`
RootNamespaceID string `json:"root_namespace_id"`
HomeNamespaceID string `json:"home_namespace_id"`
} `json:"root_info"`
}
type TokenError struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
type File struct {
Tag string `json:".tag"`
Name string `json:"name"`
PathLower string `json:"path_lower"`
PathDisplay string `json:"path_display"`
ID string `json:"id"`
ClientModified time.Time `json:"client_modified,omitempty"`
ServerModified time.Time `json:"server_modified,omitempty"`
Rev string `json:"rev,omitempty"`
Size int `json:"size,omitempty"`
IsDownloadable bool `json:"is_downloadable,omitempty"`
ContentHash string `json:"content_hash,omitempty"`
}
type Files struct {
Files []File `json:"entries"`
Cursor string `json:"cursor"`
HasMore bool `json:"has_more"`
}
type Error struct {
Error struct {
Errors []struct {
Domain string `json:"domain"`
Reason string `json:"reason"`
Message string `json:"message"`
LocationType string `json:"location_type"`
Location string `json:"location"`
}
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
func fileToObj(f File) *model.ObjThumb {
logger.Info("dropbox file", zap.Any("file", f))
obj := &model.ObjThumb{
Object: model.Object{
ID: f.ID,
Name: f.Name,
Size: int64(f.Size),
Modified: f.ClientModified,
IsFolder: f.Tag == "folder",
Path: f.PathDisplay,
},
Thumbnail: model.Thumbnail{},
}
return obj
}

102
drivers/dropbox/util.go Normal file
View File

@ -0,0 +1,102 @@
package dropbox
import (
"fmt"
"net/http"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/base"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
func (d *Dropbox) getRefreshToken() error {
url := "https://api.dropbox.com/oauth2/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"code": d.Code,
"grant_type": "authorization_code",
"redirect_uri": "https://cloudoauth.files.casaos.app",
}).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)
if err != nil {
return err
}
logger.Info("get refresh token", zap.String("res", res.String()))
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.RefreshToken = resp.RefreshToken
return nil
}
func (d *Dropbox) refreshToken() error {
url := "https://api.dropbox.com/oauth2/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"refresh_token": d.RefreshToken,
"grant_type": "refresh_token",
}).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)
if err != nil {
return err
}
logger.Info("get refresh token", zap.String("res", res.String()))
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.AccessToken = resp.AccessToken
return nil
}
func (d *Dropbox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
req := base.RestyClient.R()
req.SetHeader("Authorization", "Bearer "+d.AccessToken)
req.SetHeader("Content-Type", "application/json")
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(resp)
}
var e Error
req.SetError(&e)
res, err := req.Execute(method, url)
if err != nil {
return nil, err
}
if e.Error.Code != 0 {
if e.Error.Code == 401 {
err = d.refreshToken()
if err != nil {
return nil, err
}
return d.request(url, method, callback, resp)
}
return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
}
return res.Body(), nil
}
func (d *Dropbox) getFiles(path string) ([]File, error) {
res := make([]File, 0)
var resp Files
body := base.Json{
"limit": 2000,
"path": path,
}
_, err := d.request("https://api.dropboxapi.com/2/files/list_folder", http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp)
if err != nil {
return nil, err
}
res = append(res, resp.Files...)
return res, nil
}

183
drivers/google_drive/drive.go Normal file
View File

@ -0,0 +1,183 @@
package google_drive
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/base"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
type GoogleDrive struct {
model.StorageA
Addition
AccessToken string
}
func (d *GoogleDrive) Config() driver.Config {
return config
}
func (d *GoogleDrive) GetAddition() driver.Additional {
return &d.Addition
}
func (d *GoogleDrive) Init(ctx context.Context) error {
if d.ChunkSize == 0 {
d.ChunkSize = 5
}
if len(d.RefreshToken) == 0 {
d.getRefreshToken()
}
return d.refreshToken()
}
func (d *GoogleDrive) Drop(ctx context.Context) error {
return nil
}
func (d *GoogleDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
files, err := d.getFiles(dir.GetID())
if err != nil {
return nil, err
}
return utils.SliceConvert(files, func(src File) (model.Obj, error) {
return fileToObj(src), nil
})
}
func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?includeItemsFromAllDrives=true&supportsAllDrives=true", file.GetID())
_, err := d.request(url, http.MethodGet, nil, nil)
if err != nil {
return nil, err
}
link := model.Link{
Method: http.MethodGet,
URL: url + "&alt=media",
Header: http.Header{
"Authorization": []string{"Bearer " + d.AccessToken},
},
}
return &link, nil
}
func (d *GoogleDrive) GetUserInfo(ctx context.Context) (string, error) {
url := "https://content.googleapis.com/drive/v3/about?fields=user"
user := UserInfo{}
resp, err := d.request(url, http.MethodGet, nil, &user)
if err != nil {
return "", err
}
logger.Info("resp", zap.Any("resp", resp))
return user.User.EmailAddress, nil
}
func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
data := base.Json{
"name": dirName,
"parents": []string{parentDir.GetID()},
"mimeType": "application/vnd.google-apps.folder",
}
_, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, nil)
return err
}
func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
query := map[string]string{
"addParents": dstDir.GetID(),
"removeParents": "root",
}
url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID()
_, err := d.request(url, http.MethodPatch, func(req *resty.Request) {
req.SetQueryParams(query)
}, nil)
return err
}
func (d *GoogleDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
data := base.Json{
"name": newName,
}
url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID()
_, err := d.request(url, http.MethodPatch, func(req *resty.Request) {
req.SetBody(data)
}, nil)
return err
}
func (d *GoogleDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
return errors.New("not support")
}
func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error {
url := "https://www.googleapis.com/drive/v3/files/" + obj.GetID()
_, err := d.request(url, http.MethodDelete, nil, nil)
return err
}
func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
obj := stream.GetOld()
var (
e Error
url string
data base.Json
res *resty.Response
err error
)
if obj != nil {
url = fmt.Sprintf("https://www.googleapis.com/upload/drive/v3/files/%s?uploadType=resumable&supportsAllDrives=true", obj.GetID())
data = base.Json{}
} else {
data = base.Json{
"name": stream.GetName(),
"parents": []string{dstDir.GetID()},
}
url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true"
}
req := base.NoRedirectClient.R().
SetHeaders(map[string]string{
"Authorization": "Bearer " + d.AccessToken,
"X-Upload-Content-Type": stream.GetMimetype(),
"X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10),
}).
SetError(&e).SetBody(data).SetContext(ctx)
if obj != nil {
res, err = req.Patch(url)
} else {
res, err = req.Post(url)
}
if err != nil {
return err
}
if e.Error.Code != 0 {
if e.Error.Code == 401 {
err = d.refreshToken()
if err != nil {
return err
}
return d.Put(ctx, dstDir, stream, up)
}
return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
}
putUrl := res.Header().Get("location")
if stream.GetSize() < d.ChunkSize*1024*1024 {
_, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) {
req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)).SetBody(stream.GetReadCloser())
}, nil)
} else {
err = d.chunkUpload(ctx, stream, putUrl)
}
return err
}
var _ driver.Driver = (*GoogleDrive)(nil)

35
drivers/google_drive/meta.go Normal file
View File

@ -0,0 +1,35 @@
package google_drive
import (
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/internal/op"
)
const ICONURL = "./img/driver/GoogleDrive.svg"
const CLIENTID = "921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com"
const CLIENTSECRET = "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"
type Addition struct {
driver.RootID
RefreshToken string `json:"refresh_token" required:"true" omit:"true"`
OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime" omit:"true"`
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`
ClientID string `json:"client_id" required:"true" default:"921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com" omit:"true"`
ClientSecret string `json:"client_secret" required:"true" default:"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" omit:"true"`
ChunkSize int64 `json:"chunk_size" type:"number" help:"chunk size while uploading (unit: MB)" omit:"true"`
AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=921743327851-urr4f7jjfp4ts639evqb3i4m4qb4u4cc.apps.googleusercontent.com&redirect_uri=https%3A%2F%2Fcloudoauth.files.casaos.app&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"`
Icon string `json:"icon" type:"string" default:"./img/driver/GoogleDrive.svg"`
Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`
}
var config = driver.Config{
Name: "GoogleDrive",
OnlyProxy: true,
DefaultRoot: "root",
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &GoogleDrive{}
})
}

77
drivers/google_drive/types.go Normal file
View File

@ -0,0 +1,77 @@
package google_drive
import (
"strconv"
"time"
"github.com/IceWhaleTech/CasaOS/model"
log "github.com/sirupsen/logrus"
)
type UserInfo struct {
User struct {
Kind string `json:"kind"`
DisplayName string `json:"displayName"`
PhotoLink string `json:"photoLink"`
Me bool `json:"me"`
PermissionID string `json:"permissionId"`
EmailAddress string `json:"emailAddress"`
} `json:"user"`
}
type TokenError struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
type Files struct {
NextPageToken string `json:"nextPageToken"`
Files []File `json:"files"`
}
type File struct {
Id string `json:"id"`
Name string `json:"name"`
MimeType string `json:"mimeType"`
ModifiedTime time.Time `json:"modifiedTime"`
Size string `json:"size"`
ThumbnailLink string `json:"thumbnailLink"`
ShortcutDetails struct {
TargetId string `json:"targetId"`
TargetMimeType string `json:"targetMimeType"`
} `json:"shortcutDetails"`
}
func fileToObj(f File) *model.ObjThumb {
log.Debugf("google file: %+v", f)
size, _ := strconv.ParseInt(f.Size, 10, 64)
obj := &model.ObjThumb{
Object: model.Object{
ID: f.Id,
Name: f.Name,
Size: size,
Modified: f.ModifiedTime,
IsFolder: f.MimeType == "application/vnd.google-apps.folder",
},
Thumbnail: model.Thumbnail{},
}
if f.MimeType == "application/vnd.google-apps.shortcut" {
obj.ID = f.ShortcutDetails.TargetId
obj.IsFolder = f.ShortcutDetails.TargetMimeType == "application/vnd.google-apps.folder"
}
return obj
}
type Error struct {
Error struct {
Errors []struct {
Domain string `json:"domain"`
Reason string `json:"reason"`
Message string `json:"message"`
LocationType string `json:"location_type"`
Location string `json:"location"`
}
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}

152
drivers/google_drive/util.go Normal file
View File

@ -0,0 +1,152 @@
package google_drive
import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/base"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/go-resty/resty/v2"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
)
// do others that not defined in Driver interface
func (d *GoogleDrive) getRefreshToken() error {
url := "https://www.googleapis.com/oauth2/v4/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"client_id": d.ClientID,
"client_secret": d.ClientSecret,
"code": d.Code,
"grant_type": "authorization_code",
"redirect_uri": "https://cloudoauth.files.casaos.app",
}).Post(url)
if err != nil {
return err
}
logger.Info("get refresh token", zap.String("res", res.String()))
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.RefreshToken = resp.RefreshToken
return nil
}
func (d *GoogleDrive) refreshToken() error {
url := "https://www.googleapis.com/oauth2/v4/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"client_id": d.ClientID,
"client_secret": d.ClientSecret,
"refresh_token": d.RefreshToken,
"grant_type": "refresh_token",
}).Post(url)
if err != nil {
return err
}
log.Debug(res.String())
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.AccessToken = resp.AccessToken
return nil
}
func (d *GoogleDrive) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
req := base.RestyClient.R()
req.SetHeader("Authorization", "Bearer "+d.AccessToken)
req.SetQueryParam("includeItemsFromAllDrives", "true")
req.SetQueryParam("supportsAllDrives", "true")
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(resp)
}
var e Error
req.SetError(&e)
res, err := req.Execute(method, url)
if err != nil {
return nil, err
}
if e.Error.Code != 0 {
if e.Error.Code == 401 {
err = d.refreshToken()
if err != nil {
return nil, err
}
return d.request(url, method, callback, resp)
}
return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
}
return res.Body(), nil
}
func (d *GoogleDrive) getFiles(id string) ([]File, error) {
pageToken := "first"
res := make([]File, 0)
for pageToken != "" {
if pageToken == "first" {
pageToken = ""
}
var resp Files
orderBy := "folder,name,modifiedTime desc"
if d.OrderBy != "" {
orderBy = d.OrderBy + " " + d.OrderDirection
}
query := map[string]string{
"orderBy": orderBy,
"fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken",
"pageSize": "1000",
"q": fmt.Sprintf("'%s' in parents and trashed = false", id),
//"includeItemsFromAllDrives": "true",
//"supportsAllDrives": "true",
"pageToken": pageToken,
}
_, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, &resp)
if err != nil {
return nil, err
}
pageToken = resp.NextPageToken
res = append(res, resp.Files...)
}
return res, nil
}
func (d *GoogleDrive) chunkUpload(ctx context.Context, stream model.FileStreamer, url string) error {
var defaultChunkSize = d.ChunkSize * 1024 * 1024
var finish int64 = 0
for finish < stream.GetSize() {
if utils.IsCanceled(ctx) {
return ctx.Err()
}
chunkSize := stream.GetSize() - finish
if chunkSize > defaultChunkSize {
chunkSize = defaultChunkSize
}
_, err := d.request(url, http.MethodPut, func(req *resty.Request) {
req.SetHeaders(map[string]string{
"Content-Length": strconv.FormatInt(chunkSize, 10),
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", finish, finish+chunkSize-1, stream.GetSize()),
}).SetBody(io.LimitReader(stream.GetReadCloser(), chunkSize)).SetContext(ctx)
}, nil)
if err != nil {
return err
}
finish += chunkSize
}
return nil
}

43
internal/conf/config.go Normal file
View File

@ -0,0 +1,43 @@
package conf
type Database struct {
Type string `json:"type" env:"DB_TYPE"`
Host string `json:"host" env:"DB_HOST"`
Port int `json:"port" env:"DB_PORT"`
User string `json:"user" env:"DB_USER"`
Password string `json:"password" env:"DB_PASS"`
Name string `json:"name" env:"DB_NAME"`
DBFile string `json:"db_file" env:"DB_FILE"`
TablePrefix string `json:"table_prefix" env:"DB_TABLE_PREFIX"`
SSLMode string `json:"ssl_mode" env:"DB_SSL_MODE"`
}
type Scheme struct {
Https bool `json:"https" env:"HTTPS"`
CertFile string `json:"cert_file" env:"CERT_FILE"`
KeyFile string `json:"key_file" env:"KEY_FILE"`
}
type LogConfig struct {
Enable bool `json:"enable" env:"LOG_ENABLE"`
Name string `json:"name" env:"LOG_NAME"`
MaxSize int `json:"max_size" env:"MAX_SIZE"`
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
MaxAge int `json:"max_age" env:"MAX_AGE"`
Compress bool `json:"compress" env:"COMPRESS"`
}
type Config struct {
Force bool `json:"force" env:"FORCE"`
Address string `json:"address" env:"ADDR"`
Port int `json:"port" env:"PORT"`
SiteURL string `json:"site_url" env:"SITE_URL"`
Cdn string `json:"cdn" env:"CDN"`
JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"`
TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
Database Database `json:"database"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
Log LogConfig `json:"log"`
}

72
internal/conf/const.go Normal file
View File

@ -0,0 +1,72 @@
package conf
const (
TypeString = "string"
TypeSelect = "select"
TypeBool = "bool"
TypeText = "text"
TypeNumber = "number"
)
const (
// site
VERSION = "version"
ApiUrl = "api_url"
BasePath = "base_path"
SiteTitle = "site_title"
Announcement = "announcement"
AllowIndexed = "allow_indexed"
Logo = "logo"
Favicon = "favicon"
MainColor = "main_color"
// preview
TextTypes = "text_types"
AudioTypes = "audio_types"
VideoTypes = "video_types"
ImageTypes = "image_types"
ProxyTypes = "proxy_types"
ProxyIgnoreHeaders = "proxy_ignore_headers"
AudioAutoplay = "audio_autoplay"
VideoAutoplay = "video_autoplay"
// global
HideFiles = "hide_files"
CustomizeHead = "customize_head"
CustomizeBody = "customize_body"
LinkExpiration = "link_expiration"
SignAll = "sign_all"
PrivacyRegs = "privacy_regs"
OcrApi = "ocr_api"
FilenameCharMapping = "filename_char_mapping"
// index
SearchIndex = "search_index"
AutoUpdateIndex = "auto_update_index"
IndexPaths = "index_paths"
IgnorePaths = "ignore_paths"
// aria2
Aria2Uri = "aria2_uri"
Aria2Secret = "aria2_secret"
// single
Token = "token"
IndexProgress = "index_progress"
//Github
GithubClientId = "github_client_id"
GithubClientSecrets = "github_client_secrets"
GithubLoginEnabled = "github_login_enabled"
)
const (
UNKNOWN = iota
FOLDER
//OFFICE
VIDEO
AUDIO
TEXT
IMAGE
)

30
internal/conf/var.go Normal file
View File

@ -0,0 +1,30 @@
package conf
import "regexp"
var (
BuiltAt string
GoVersion string
GitAuthor string
GitCommit string
Version string = "dev"
WebVersion string
)
var (
Conf *Config
)
var SlicesMap = make(map[string][]string)
var FilenameCharMap = make(map[string]string)
var PrivacyReg []*regexp.Regexp
var (
// StoragesLoaded loaded success if empty
StoragesLoaded = false
)
var (
RawIndexHtml string
ManageHtml string
IndexHtml string
)

25
internal/driver/config.go Normal file
View File

@ -0,0 +1,25 @@
/*
* @Author: a624669980@163.com a624669980@163.com
* @Date: 2022-12-13 11:05:05
* @LastEditors: a624669980@163.com a624669980@163.com
* @LastEditTime: 2022-12-13 11:05:13
* @FilePath: /drive/internal/driver/config.go
* @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE
*/
package driver
type Config struct {
Name string `json:"name"`
LocalSort bool `json:"local_sort"`
OnlyLocal bool `json:"only_local"`
OnlyProxy bool `json:"only_proxy"`
NoCache bool `json:"no_cache"`
NoUpload bool `json:"no_upload"`
NeedMs bool `json:"need_ms"` // if need get message from user, such as validate code
DefaultRoot string `json:"default_root"`
CheckStatus bool
}
func (c Config) MustProxy() bool {
return c.OnlyProxy || c.OnlyLocal
}

131
internal/driver/driver.go Normal file
View File

@ -0,0 +1,131 @@
package driver
import (
"context"
"github.com/IceWhaleTech/CasaOS/model"
)
type Driver interface {
Meta
Reader
User
//Writer
//Other
}
type Meta interface {
Config() Config
// GetStorage just get raw storage, no need to implement, because model.Storage have implemented
GetStorage() *model.StorageA
SetStorage(model.StorageA)
// GetAddition Additional is used for unmarshal of JSON, so need return pointer
GetAddition() Additional
// Init If already initialized, drop first
Init(ctx context.Context) error
Drop(ctx context.Context) error
}
type Other interface {
Other(ctx context.Context, args model.OtherArgs) (interface{}, error)
}
type Reader interface {
// List files in the path
// if identify files by path, need to set ID with path,like path.Join(dir.GetID(), obj.GetName())
// if identify files by id, need to set ID with corresponding id
List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error)
// Link get url/filepath/reader of file
Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error)
}
type User interface {
// GetRoot get root directory of user
GetUserInfo(ctx context.Context) (string, error)
}
type Getter interface {
GetRoot(ctx context.Context) (model.Obj, error)
}
//type Writer interface {
// Mkdir
// Move
// Rename
// Copy
// Remove
// Put
//}
type Mkdir interface {
MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error
}
type Move interface {
Move(ctx context.Context, srcObj, dstDir model.Obj) error
}
type Rename interface {
Rename(ctx context.Context, srcObj model.Obj, newName string) error
}
type Copy interface {
Copy(ctx context.Context, srcObj, dstDir model.Obj) error
}
type Remove interface {
Remove(ctx context.Context, obj model.Obj) error
}
type Put interface {
Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error
}
//type WriteResult interface {
// MkdirResult
// MoveResult
// RenameResult
// CopyResult
// PutResult
// Remove
//}
type MkdirResult interface {
MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error)
}
type MoveResult interface {
Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error)
}
type RenameResult interface {
Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error)
}
type CopyResult interface {
Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error)
}
type PutResult interface {
Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error)
}
type UpdateProgress func(percentage int)
type Progress struct {
Total int64
Done int64
up UpdateProgress
}
func (p *Progress) Write(b []byte) (n int, err error) {
n = len(b)
p.Done += int64(n)
p.up(int(float64(p.Done) / float64(p.Total) * 100))
return
}
func NewProgress(total int64, up UpdateProgress) *Progress {
return &Progress{
Total: total,
up: up,
}
}

56
internal/driver/item.go Normal file
View File

@ -0,0 +1,56 @@
/*
* @Author: a624669980@163.com a624669980@163.com
* @Date: 2022-12-13 11:05:47
* @LastEditors: a624669980@163.com a624669980@163.com
* @LastEditTime: 2022-12-13 11:05:54
* @FilePath: /drive/internal/driver/item.go
* @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE
*/
package driver
type Additional interface{}
type Select string
type Item struct {
Name string `json:"name"`
Type string `json:"type"`
Default string `json:"default"`
Options string `json:"options"`
Required bool `json:"required"`
Help string `json:"help"`
}
type Info struct {
Common []Item `json:"common"`
Additional []Item `json:"additional"`
Config Config `json:"config"`
}
type IRootPath interface {
GetRootPath() string
}
type IRootId interface {
GetRootId() string
}
type RootPath struct {
RootFolderPath string `json:"root_folder_path"`
}
type RootID struct {
RootFolderID string `json:"root_folder_id" omit:"true"`
}
func (r RootPath) GetRootPath() string {
return r.RootFolderPath
}
func (r *RootPath) SetRootPath(path string) {
r.RootFolderPath = path
}
func (r RootID) GetRootId() string {
return r.RootFolderID
}

6
internal/op/const.go Normal file
View File

@ -0,0 +1,6 @@
package op
const (
WORK = "work"
RootName = "root"
)

173
internal/op/driver.go Normal file
View File

@ -0,0 +1,173 @@
package op
import (
"reflect"
"strings"
"github.com/IceWhaleTech/CasaOS/internal/conf"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/pkg/errors"
)
type New func() driver.Driver
var driverNewMap = map[string]New{}
var driverInfoMap = map[string][]driver.Item{} //driver.Info{}
func RegisterDriver(driver New) {
// log.Infof("register driver: [%s]", config.Name)
tempDriver := driver()
tempConfig := tempDriver.Config()
registerDriverItems(tempConfig, tempDriver.GetAddition())
driverNewMap[tempConfig.Name] = driver
}
func GetDriverNew(name string) (New, error) {
n, ok := driverNewMap[name]
if !ok {
return nil, errors.Errorf("no driver named: %s", name)
}
return n, nil
}
func GetDriverNames() []string {
var driverNames []string
for k := range driverInfoMap {
driverNames = append(driverNames, k)
}
return driverNames
}
// func GetDriverInfoMap() map[string]driver.Info {
// return driverInfoMap
// }
func GetDriverInfoMap() map[string][]driver.Item {
return driverInfoMap
}
func registerDriverItems(config driver.Config, addition driver.Additional) {
// log.Debugf("addition of %s: %+v", config.Name, addition)
tAddition := reflect.TypeOf(addition)
for tAddition.Kind() == reflect.Pointer {
tAddition = tAddition.Elem()
}
//mainItems := getMainItems(config)
additionalItems := getAdditionalItems(tAddition, config.DefaultRoot)
driverInfoMap[config.Name] = additionalItems
// driver.Info{
// Common: mainItems,
// Additional: additionalItems,
// Config: config,
// }
}
func getMainItems(config driver.Config) []driver.Item {
items := []driver.Item{{
Name: "mount_path",
Type: conf.TypeString,
Required: true,
Help: "",
}, {
Name: "order",
Type: conf.TypeNumber,
Help: "use to sort",
}, {
Name: "remark",
Type: conf.TypeText,
}}
if !config.NoCache {
items = append(items, driver.Item{
Name: "cache_expiration",
Type: conf.TypeNumber,
Default: "30",
Required: true,
Help: "The cache expiration time for this storage",
})
}
if !config.OnlyProxy && !config.OnlyLocal {
items = append(items, []driver.Item{{
Name: "web_proxy",
Type: conf.TypeBool,
}, {
Name: "webdav_policy",
Type: conf.TypeSelect,
Options: "302_redirect,use_proxy_url,native_proxy",
Default: "302_redirect",
Required: true,
},
}...)
} else {
items = append(items, driver.Item{
Name: "webdav_policy",
Type: conf.TypeSelect,
Default: "native_proxy",
Options: "use_proxy_url,native_proxy",
Required: true,
})
}
items = append(items, driver.Item{
Name: "down_proxy_url",
Type: conf.TypeText,
})
if config.LocalSort {
items = append(items, []driver.Item{{
Name: "order_by",
Type: conf.TypeSelect,
Options: "name,size,modified",
}, {
Name: "order_direction",
Type: conf.TypeSelect,
Options: "asc,desc",
}}...)
}
items = append(items, driver.Item{
Name: "extract_folder",
Type: conf.TypeSelect,
Options: "front,back",
})
return items
}
func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {
var items []driver.Item
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if field.Type.Kind() == reflect.Struct {
items = append(items, getAdditionalItems(field.Type, defaultRoot)...)
continue
}
tag := field.Tag
ignore, ok1 := tag.Lookup("ignore")
name, ok2 := tag.Lookup("json")
if (ok1 && ignore == "true") || !ok2 {
continue
}
if tag.Get("omit") == "true" {
continue
}
item := driver.Item{
Name: name,
Type: strings.ToLower(field.Type.Name()),
Default: tag.Get("default"),
Options: tag.Get("options"),
Required: tag.Get("required") == "true",
Help: tag.Get("help"),
}
if tag.Get("type") != "" {
item.Type = tag.Get("type")
}
if item.Name == "root_folder_id" || item.Name == "root_folder_path" {
if item.Default == "" {
item.Default = defaultRoot
}
item.Required = item.Default != ""
}
// set default type to string
if item.Type == "" {
item.Type = "string"
}
items = append(items, item)
}
return items
}

545
internal/op/fs.go Normal file
View File

@ -0,0 +1,545 @@
package op
import (
"context"
"os"
stdpath "path"
"time"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/generic_sync"
"github.com/IceWhaleTech/CasaOS/pkg/singleflight"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/Xhofe/go-cache"
"github.com/pkg/errors"
pkgerr "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// In order to facilitate adding some other things before and after file op
var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var listG singleflight.Group[[]model.Obj]
func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == oldObj.GetName() {
objs[i] = newObj
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, oldObj := range objs {
if oldObj.GetName() == obj.GetName() {
objs = append(objs[:i], objs[i+1:]...)
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == newObj.GetName() {
objs[i] = newObj
return
}
}
// Simple separation of files and folders
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
objs = append(objs, newObj)
} else {
objs = append([]model.Obj{newObj}, objs...)
}
if storage.Config().LocalSort {
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
log.Debug("addCacheObj: wait start sort")
debounce(func() {
log.Debug("addCacheObj: start sort")
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
addSortDebounceMap.Delete(key)
})
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func ClearCache(storage driver.Driver, path string) {
listCache.Del(Key(storage, path))
}
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))
}
// List files in storage, not contains virtual file
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
log.Debugf("op.List %s", path)
key := Key(storage, path)
if !utils.IsBool(refresh...) {
if files, ok := listCache.Get(key); ok {
log.Debugf("use cache when list %s", path)
return files, nil
}
}
dir, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
}
log.Debugf("list dir: %+v", dir)
if !dir.IsDir() {
return nil, errors.WithStack(errors.New("not a folder"))
}
objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
files, err := storage.List(ctx, dir, args)
if err != nil {
return nil, errors.Wrapf(err, "failed to list objs")
}
// set path
for _, f := range files {
if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" {
s.SetPath(stdpath.Join(dir.GetPath(), f.GetName()))
}
}
// warp obj name
model.WrapObjsName(files)
// call hooks
go func(reqPath string, files []model.Obj) {
for _, hook := range ObjsUpdateHooks {
hook(args.ReqPath, files)
}
}(args.ReqPath, files)
// sort objs
if storage.Config().LocalSort {
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
} else {
log.Debugf("del cache: %s", key)
listCache.Del(key)
}
}
return files, nil
})
return objs, err
}
// Get object from list of files
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
path = utils.FixAndCleanPath(path)
log.Debugf("op.Get %s", path)
// is root folder
if utils.PathEqual(path, "/") {
var rootObj model.Obj
switch r := storage.GetAddition().(type) {
case driver.IRootId:
rootObj = &model.Object{
ID: r.GetRootId(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
Path: path,
}
case driver.IRootPath:
rootObj = &model.Object{
Path: r.GetRootPath(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
}
default:
if storage, ok := storage.(driver.Getter); ok {
obj, err := storage.GetRoot(ctx)
if err != nil {
return nil, errors.WithMessage(err, "failed get root obj")
}
rootObj = obj
}
}
if rootObj == nil {
return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method")
}
return &model.ObjWrapName{
Name: RootName,
Obj: rootObj,
}, nil
}
// not root folder
dir, name := stdpath.Split(path)
files, err := List(ctx, storage, dir, model.ListArgs{})
if err != nil {
return nil, errors.WithMessage(err, "failed get parent list")
}
for _, f := range files {
// TODO maybe copy obj here
if f.GetName() == name {
return f, nil
}
}
log.Debugf("cant find obj with name: %s", name)
return nil, errors.WithStack(errors.New("object not found"))
}
func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
obj, err := Get(ctx, storage, path)
if err != nil {
return nil, err
}
return model.UnwrapObjs(obj), err
}
var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
var linkG singleflight.Group[*model.Link]
// Link get link, if is an url. should have an expiry time
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
file, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed to get file")
}
if file.IsDir() {
return nil, nil, errors.WithStack(errors.New("not a file"))
}
key := Key(storage, path) + ":" + args.IP
if link, ok := linkCache.Get(key); ok {
return link, file, nil
}
fn := func() (*model.Link, error) {
link, err := storage.Link(ctx, file, args)
if err != nil {
return nil, errors.Wrapf(err, "failed get link")
}
if link.Expiration != nil {
linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))
}
return link, nil
}
link, err, _ := linkG.Do(key, fn)
return link, file, err
}
// Other api
func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) {
obj, err := GetUnwrap(ctx, storage, args.Path)
if err != nil {
return nil, errors.WithMessagef(err, "failed to get obj")
}
if o, ok := storage.(driver.Other); ok {
return o.Other(ctx, model.OtherArgs{
Obj: obj,
Method: args.Method,
Data: args.Data,
})
} else {
return nil, errors.New("not implement")
}
}
var mkdirG singleflight.Group[interface{}]
func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
key := Key(storage, path)
_, err, _ := mkdirG.Do(key, func() (interface{}, error) {
// check if dir exists
f, err := GetUnwrap(ctx, storage, path)
if err != nil {
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
parentPath, dirName := stdpath.Split(path)
err = MakeDir(ctx, storage, parentPath)
if err != nil {
return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
parentDir, err := GetUnwrap(ctx, storage, parentPath)
// this should not happen
if err != nil {
return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
switch s := storage.(type) {
case driver.MkdirResult:
var newObj model.Obj
newObj, err = s.MakeDir(ctx, parentDir, dirName)
if err == nil {
if newObj != nil {
addCacheObj(storage, parentPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
}
case driver.Mkdir:
err = s.MakeDir(ctx, parentDir, dirName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
default:
return nil, errors.New("not implement")
}
return nil, errors.WithStack(err)
}
return nil, errors.WithMessage(err, "failed to check if dir exists")
}
// dir exists
if f.IsDir() {
return nil, nil
}
// dir to make is a file
return nil, errors.New("file exists")
})
return err
}
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.MoveResult:
var newObj model.Obj
newObj, err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Move:
err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.RenameResult:
var newObj model.Obj
newObj, err = s.Rename(ctx, srcObj, dstName)
if err == nil {
if newObj != nil {
updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
}
case driver.Rename:
err = s.Rename(ctx, srcObj, dstName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
// Copy Just copy file[s] in a storage
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcObj, err := GetUnwrap(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
switch s := storage.(type) {
case driver.CopyResult:
var newObj model.Obj
newObj, err = s.Copy(ctx, srcObj, dstDir)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Copy:
err = s.Copy(ctx, srcObj, dstDir)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Remove(ctx context.Context, storage driver.Driver, path string) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
rawObj, err := Get(ctx, storage, path)
if err != nil {
// if object not found, it's ok
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
return nil
}
return errors.WithMessage(err, "failed to get object")
}
dirPath := stdpath.Dir(path)
switch s := storage.(type) {
case driver.Remove:
err = s.Remove(ctx, model.UnwrapObjs(rawObj))
if err == nil {
delCacheObj(storage, dirPath, rawObj)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
defer func() {
if f, ok := file.GetReadCloser().(*os.File); ok {
err := os.RemoveAll(f.Name())
if err != nil {
log.Errorf("failed to remove file [%s]", f.Name())
}
}
}()
defer func() {
if err := file.Close(); err != nil {
log.Errorf("failed to close file streamer, %v", err)
}
}()
// if file exist and size = 0, delete it
dstDirPath = utils.FixAndCleanPath(dstDirPath)
dstPath := stdpath.Join(dstDirPath, file.GetName())
fi, err := GetUnwrap(ctx, storage, dstPath)
if err == nil {
if fi.GetSize() == 0 {
err = Remove(ctx, storage, dstPath)
if err != nil {
return errors.WithMessagef(err, "failed remove file that exist and have size 0")
}
} else {
file.Old = fi
}
}
err = MakeDir(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
}
parentDir, err := GetUnwrap(ctx, storage, dstDirPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
}
// if up is nil, set a default to prevent panic
if up == nil {
up = func(p int) {}
}
switch s := storage.(type) {
case driver.PutResult:
var newObj model.Obj
newObj, err = s.Put(ctx, parentDir, file, up)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Put:
err = s.Put(ctx, parentDir, file, up)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
log.Debugf("put file [%s] done", file.GetName())
//if err == nil {
// //clear cache
// key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
// listCache.Del(key)
//}
return errors.WithStack(err)
}

109
internal/op/hook.go Normal file
View File

@ -0,0 +1,109 @@
package op
import (
"regexp"
"strings"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/internal/conf"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
"go.uber.org/zap"
)
// Obj
type ObjsUpdateHook = func(parent string, objs []model.Obj)
var (
ObjsUpdateHooks = make([]ObjsUpdateHook, 0)
)
func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
ObjsUpdateHooks = append(ObjsUpdateHooks, hook)
}
func HandleObjsUpdateHook(parent string, objs []model.Obj) {
for _, hook := range ObjsUpdateHooks {
hook(parent, objs)
}
}
// Setting
type SettingItemHook func(item *model.SettingItem) error
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.SlicesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.SlicesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.SlicesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.SlicesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.SlicesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyIgnoreHeaders: func(item *model.SettingItem) error {
conf.SlicesMap[conf.ProxyIgnoreHeaders] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
err := json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
logger.Info("filename char mapping", zap.Any("FilenameCharMap", conf.FilenameCharMap))
return nil
},
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) {
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
// Storage
type StorageHook func(typ string, storage driver.Driver)
var storageHooks = make([]StorageHook, 0)
func CallStorageHooks(typ string, storage driver.Driver) {
for _, hook := range storageHooks {
hook(typ, storage)
}
}
func RegisterStorageHook(hook StorageHook) {
storageHooks = append(storageHooks, hook)
}

36
internal/sign/sign.go Normal file
View File

@ -0,0 +1,36 @@
package sign
import (
"sync"
"time"
"github.com/IceWhaleTech/CasaOS/pkg/sign"
)
var once sync.Once
var instance sign.Sign
func Sign(data string) string {
return NotExpired(data)
}
func WithDuration(data string, d time.Duration) string {
once.Do(Instance)
return instance.Sign(data, time.Now().Add(d).Unix())
}
func NotExpired(data string) string {
once.Do(Instance)
return instance.Sign(data, 0)
}
func Verify(data string, sign string) error {
once.Do(Instance)
return instance.Verify(data, sign)
}
func Instance() {
instance = sign.NewHMACSign([]byte("token"))
}

12
main.go
View File

@ -80,6 +80,11 @@ func init() {
service.GetCPUThermalZone()
route.InitFunction()
///
// service.MountLists = make(map[string]*mountlib.MountPoint)
// configfile.Install()
service.MyService.Storage().CheckAndMountAll()
}
// @title casaOS API
@ -141,9 +146,9 @@ func main() {
"/v1/image",
"/v1/samba",
"/v1/notify",
//"/v1/driver",
//"/v1/cloud",
//"/v1/recover",
"/v1/driver",
"/v1/cloud",
"/v1/recover",
"/v1/other",
route.V2APIPath,
route.V2DocPath,
@ -162,6 +167,7 @@ func main() {
}
var events []message_bus.EventType
events = append(events, message_bus.EventType{Name: "casaos:system:utilization", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}})
events = append(events, message_bus.EventType{Name: "casaos:file:recover", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}})
events = append(events, message_bus.EventType{Name: "casaos:file:operate", SourceID: common.SERVICENAME, PropertyTypeList: []message_bus.PropertyType{}})
// register at message bus
for i := 0; i < 10; i++ {

8
model/storage.go
View File

@ -2,7 +2,7 @@ package model
import "time"
type Storage struct {
type StorageA struct {
ID uint `json:"id" gorm:"primaryKey"` // unique key
MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized
Order int `json:"order"` // use to sort
@ -29,15 +29,15 @@ type Proxy struct {
DownProxyUrl string `json:"down_proxy_url"`
}
func (s *Storage) GetStorage() *Storage {
func (s *StorageA) GetStorage() *StorageA {
return s
}
func (s *Storage) SetStorage(storage Storage) {
func (s *StorageA) SetStorage(storage StorageA) {
*s = storage
}
func (s *Storage) SetStatus(status string) {
func (s *StorageA) SetStatus(status string) {
s.Status = status
}

235
pkg/utils/httper/drive.go
View File

@ -1,7 +1,15 @@
package httper
import (
"encoding/json"
"fmt"
"net"
"net/http"
"time"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
type MountList struct {
@ -36,126 +44,127 @@ type RemotesResult struct {
var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
var DefaultTimeout = time.Second * 30
// func NewRestyClient() *resty.Client {
func NewRestyClient() *resty.Client {
// unixSocket := "/var/run/rclone/rclone.sock"
unixSocket := "/var/run/rclone/rclone.sock"
// transport := http.Transport{
// Dial: func(_, _ string) (net.Conn, error) {
// return net.Dial("unix", unixSocket)
// },
// }
transport := http.Transport{
Dial: func(_, _ string) (net.Conn, error) {
return net.Dial("unix", unixSocket)
},
}
// client := resty.New()
client := resty.New()
// client.SetTransport(&transport).SetBaseURL("http://localhost")
// client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent)
// return client
// }
client.SetTransport(&transport).SetBaseURL("http://localhost")
client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent)
return client
}
// func GetMountList() (MountList, error) {
// var result MountList
// res, err := NewRestyClient().R().Post("/mount/listmounts")
// if err != nil {
// return result, err
// }
// if res.StatusCode() != 200 {
// return result, fmt.Errorf("get mount list failed")
// }
// json.Unmarshal(res.Body(), &result)
// for i := 0; i < len(result.MountPoints); i++ {
// result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1]
// }
// return result, err
// }
//
// func Mount(mountPoint string, fs string) error {
// res, err := NewRestyClient().R().SetFormData(map[string]string{
// "mountPoint": mountPoint,
// "fs": fs,
// "mountOpt": `{"AllowOther": true}`,
// }).Post("/mount/mount")
// if err != nil {
// return err
// }
// if res.StatusCode() != 200 {
// return fmt.Errorf("mount failed")
// }
// logger.Info("mount then", zap.Any("res", res.Body()))
// return nil
// }
// func Unmount(mountPoint string) error {
// res, err := NewRestyClient().R().SetFormData(map[string]string{
// "mountPoint": mountPoint,
// }).Post("/mount/unmount")
// if err != nil {
// logger.Error("when unmount", zap.Error(err))
// return err
// }
// if res.StatusCode() != 200 {
// logger.Error("then unmount failed", zap.Any("res", res.Body()))
// return fmt.Errorf("unmount failed")
// }
// logger.Info("unmount then", zap.Any("res", res.Body()))
// return nil
// }
func GetMountList() (MountList, error) {
var result MountList
res, err := NewRestyClient().R().Post("/mount/listmounts")
if err != nil {
return result, err
}
if res.StatusCode() != 200 {
return result, fmt.Errorf("get mount list failed")
}
json.Unmarshal(res.Body(), &result)
for i := 0; i < len(result.MountPoints); i++ {
result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1]
}
return result, err
}
// func CreateConfig(data map[string]string, name, t string) error {
// data["config_is_local"] = "false"
// dataStr, _ := json.Marshal(data)
// res, err := NewRestyClient().R().SetFormData(map[string]string{
// "name": name,
// "parameters": string(dataStr),
// "type": t,
// }).Post("/config/create")
// logger.Info("when create config then", zap.Any("res", res.Body()))
// if err != nil {
// return err
// }
// if res.StatusCode() != 200 {
// return fmt.Errorf("create config failed")
// }
func Mount(mountPoint string, fs string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"mountPoint": mountPoint,
"fs": fs,
"mountOpt": `{"AllowOther": true}`,
}).Post("/mount/mount")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("mount failed")
}
logger.Info("mount then", zap.Any("res", res.Body()))
return nil
}
func Unmount(mountPoint string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"mountPoint": mountPoint,
}).Post("/mount/unmount")
if err != nil {
logger.Error("when unmount", zap.Error(err))
return err
}
if res.StatusCode() != 200 {
logger.Error("then unmount failed", zap.Any("res", res.Body()))
return fmt.Errorf("unmount failed")
}
// return nil
// }
logger.Info("unmount then", zap.Any("res", res.Body()))
return nil
}
// func GetConfigByName(name string) (map[string]string, error) {
func CreateConfig(data map[string]string, name, t string) error {
data["config_is_local"] = "false"
dataStr, _ := json.Marshal(data)
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
"parameters": string(dataStr),
"type": t,
}).Post("/config/create")
logger.Info("when create config then", zap.Any("res", res.Body()))
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("create config failed")
}
// res, err := NewRestyClient().R().SetFormData(map[string]string{
// "name": name,
// }).Post("/config/get")
// if err != nil {
// return nil, err
// }
// if res.StatusCode() != 200 {
// return nil, fmt.Errorf("create config failed")
// }
// var result map[string]string
// json.Unmarshal(res.Body(), &result)
// return result, nil
// }
// func GetAllConfigName() (RemotesResult, error) {
// var result RemotesResult
// res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes")
// if err != nil {
// return result, err
// }
// if res.StatusCode() != 200 {
// return result, fmt.Errorf("get config failed")
// }
return nil
}
// json.Unmarshal(res.Body(), &result)
// return result, nil
// }
// func DeleteConfigByName(name string) error {
// res, err := NewRestyClient().R().SetFormData(map[string]string{
// "name": name,
// }).Post("/config/delete")
// if err != nil {
// return err
// }
// if res.StatusCode() != 200 {
// return fmt.Errorf("delete config failed")
// }
// return nil
// }
func GetConfigByName(name string) (map[string]string, error) {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
}).Post("/config/get")
if err != nil {
return nil, err
}
if res.StatusCode() != 200 {
return nil, fmt.Errorf("create config failed")
}
var result map[string]string
json.Unmarshal(res.Body(), &result)
return result, nil
}
func GetAllConfigName() (RemotesResult, error) {
var result RemotesResult
res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes")
if err != nil {
return result, err
}
if res.StatusCode() != 200 {
return result, fmt.Errorf("get config failed")
}
json.Unmarshal(res.Body(), &result)
return result, nil
}
func DeleteConfigByName(name string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
}).Post("/config/delete")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("delete config failed")
}
return nil
}

13
route/v1.go
View File

@ -36,7 +36,7 @@ func InitV1Router() *gin.Engine {
r.GET("/ping", func(ctx *gin.Context) {
ctx.String(200, "pong")
})
r.GET("/v1/recover/:type", v1.GetRecoverStorage)
v1Group := r.Group("/v1")
v1Group.Use(jwt.ExceptLocalhost())
@ -98,6 +98,17 @@ func InitV1Router() *gin.Engine {
v1FileGroup.GET("/ws", v1.ConnectWebSocket)
v1FileGroup.GET("/peers", v1.GetPeers)
}
v1CloudGroup := v1Group.Group("/cloud")
v1CloudGroup.Use()
{
v1CloudGroup.GET("", v1.ListStorages)
v1CloudGroup.DELETE("", v1.UmountStorage)
}
v1DriverGroup := v1Group.Group("/driver")
v1DriverGroup.Use()
{
v1DriverGroup.GET("", v1.ListDriverInfo)
}
v1FolderGroup := v1Group.Group("/folder")
v1FolderGroup.Use()

101
route/v1/cloud.go Normal file
View File

@ -0,0 +1,101 @@
package v1
import (
"strings"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/dropbox"
"github.com/IceWhaleTech/CasaOS/drivers/google_drive"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils/common_err"
"github.com/IceWhaleTech/CasaOS/pkg/utils/httper"
"github.com/IceWhaleTech/CasaOS/service"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
func ListStorages(c *gin.Context) {
// var req model.PageReq
// if err := c.ShouldBind(&req); err != nil {
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})
// return
// }
// req.Validate()
//logger.Info("ListStorages", zap.Any("req", req))
//storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage)
// if err != nil {
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
// return
// }
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{
// Content: storages,
// Total: total,
// }})
r, err := service.MyService.Storage().GetStorages()
if err != nil {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return
}
for i := 0; i < len(r.MountPoints); i++ {
dataMap, err := service.MyService.Storage().GetConfigByName(r.MountPoints[i].Fs)
if err != nil {
logger.Error("GetConfigByName", zap.Any("err", err))
continue
}
if dataMap["type"] == "drive" {
r.MountPoints[i].Icon = google_drive.ICONURL
}
if dataMap["type"] == "dropbox" {
r.MountPoints[i].Icon = dropbox.ICONURL
}
r.MountPoints[i].Name = dataMap["username"]
}
list := []httper.MountPoint{}
for _, v := range r.MountPoints {
list = append(list, httper.MountPoint{
Fs: v.Fs,
Icon: v.Icon,
MountPoint: v.MountPoint,
Name: v.Name,
})
}
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: list})
}
func UmountStorage(c *gin.Context) {
json := make(map[string]string)
c.ShouldBind(&json)
mountPoint := json["mount_point"]
if mountPoint == "" {
c.JSON(common_err.CLIENT_ERROR, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: "mount_point is empty"})
return
}
err := service.MyService.Storage().UnmountStorage(mountPoint)
if err != nil {
c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return
}
service.MyService.Storage().DeleteConfigByName(strings.ReplaceAll(mountPoint, "/mnt/", ""))
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"})
}
func GetStorage(c *gin.Context) {
// idStr := c.Query("id")
// id, err := strconv.Atoi(idStr)
// if err != nil {
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})
// return
// }
// storage, err := service.MyService.Storage().GetStorageById(uint(id))
// if err != nil {
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
// return
// }
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage})
}

12
route/v1/driver.go Normal file
View File

@ -0,0 +1,12 @@
package v1
import (
"github.com/IceWhaleTech/CasaOS-Common/model"
"github.com/IceWhaleTech/CasaOS-Common/utils/common_err"
"github.com/IceWhaleTech/CasaOS/internal/op"
"github.com/gin-gonic/gin"
)
func ListDriverInfo(c *gin.Context) {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: op.GetDriverInfoMap()})
}

205
route/v1/recover.go Normal file
View File

@ -0,0 +1,205 @@
package v1
import (
"strconv"
"strings"
"time"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/dropbox"
"github.com/IceWhaleTech/CasaOS/drivers/google_drive"
"github.com/IceWhaleTech/CasaOS/service"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
func GetRecoverStorage(c *gin.Context) {
c.Header("Content-Type", "text/html; charset=utf-8")
t := c.Param("type")
currentTime := time.Now().UTC()
currentDate := time.Now().UTC().Format("2006-01-02")
notify := make(map[string]interface{})
if t == "GoogleDrive" {
add := google_drive.Addition{}
add.Code = c.Query("code")
if len(add.Code) == 0 {
c.String(200, `<p>Code cannot be empty</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Code cannot be empty"
logger.Error("Then code is empty: ", zap.String("code", add.Code), zap.Any("name", "google_drive"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
add.RootFolderID = "root"
add.ClientID = google_drive.CLIENTID
add.ClientSecret = google_drive.CLIENTSECRET
var google_drive google_drive.GoogleDrive
google_drive.Addition = add
err := google_drive.Init(c)
if err != nil {
c.String(200, `<p>Initialization failure:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Initialization failure"
logger.Error("Then init error: ", zap.Error(err), zap.Any("name", "google_drive"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
username, err := google_drive.GetUserInfo(c)
if err != nil {
c.String(200, `<p>Failed to get user information:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Failed to get user information"
logger.Error("Then get user info error: ", zap.Error(err), zap.Any("name", "google_drive"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
dmap := make(map[string]string)
dmap["username"] = username
configs, err := service.MyService.Storage().GetConfig()
if err != nil {
c.String(200, `<p>Failed to get rclone config:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Failed to get rclone config"
logger.Error("Then get config error: ", zap.Error(err), zap.Any("name", "google_drive"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
for _, v := range configs.Remotes {
cf, err := service.MyService.Storage().GetConfigByName(v)
if err != nil {
logger.Error("then get config by name error: ", zap.Error(err), zap.Any("name", v))
continue
}
if cf["type"] == "drive" && cf["username"] == dmap["username"] {
c.String(200, `<p>The same configuration has been added</p><script>window.close()</script>`)
err := service.MyService.Storage().CheckAndMountByName(v)
if err != nil {
logger.Error("check and mount by name error: ", zap.Error(err), zap.Any("name", cf["username"]))
}
notify["status"] = "warn"
notify["message"] = "The same configuration has been added"
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
}
if len(username) > 0 {
a := strings.Split(username, "@")
username = a[0]
}
//username = fileutil.NameAccumulation(username, "/mnt")
username += "_google_drive_" + strconv.FormatInt(time.Now().Unix(), 10)
dmap["client_id"] = add.ClientID
dmap["client_secret"] = add.ClientSecret
dmap["scope"] = "drive"
dmap["mount_point"] = "/mnt/" + username
dmap["token"] = `{"access_token":"` + google_drive.AccessToken + `","token_type":"Bearer","refresh_token":"` + google_drive.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*1).Add(time.Minute*50).Format("15:04:05") + `Z"}`
service.MyService.Storage().CreateConfig(dmap, username, "drive")
service.MyService.Storage().MountStorage("/mnt/"+username, username+":")
notify := make(map[string]interface{})
notify["status"] = "success"
notify["message"] = "Success"
notify["driver"] = "GoogleDrive"
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
} else if t == "Dropbox" {
add := dropbox.Addition{}
add.Code = c.Query("code")
if len(add.Code) == 0 {
c.String(200, `<p>Code cannot be empty</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Code cannot be empty"
logger.Error("Then code is empty error: ", zap.String("code", add.Code), zap.Any("name", "dropbox"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
add.RootFolderID = ""
add.AppKey = dropbox.APPKEY
add.AppSecret = dropbox.APPSECRET
var dropbox dropbox.Dropbox
dropbox.Addition = add
err := dropbox.Init(c)
if err != nil {
c.String(200, `<p>Initialization failure:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Initialization failure"
logger.Error("Then init error: ", zap.Error(err), zap.Any("name", "dropbox"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
username, err := dropbox.GetUserInfo(c)
if err != nil {
c.String(200, `<p>Failed to get user information:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Failed to get user information"
logger.Error("Then get user information: ", zap.Error(err), zap.Any("name", "dropbox"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
dmap := make(map[string]string)
dmap["username"] = username
configs, err := service.MyService.Storage().GetConfig()
if err != nil {
c.String(200, `<p>Failed to get rclone config:`+err.Error()+`</p><script>window.close()</script>`)
notify["status"] = "fail"
notify["message"] = "Failed to get rclone config"
logger.Error("Then get config error: ", zap.Error(err), zap.Any("name", "dropbox"))
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
for _, v := range configs.Remotes {
cf, err := service.MyService.Storage().GetConfigByName(v)
if err != nil {
logger.Error("then get config by name error: ", zap.Error(err), zap.Any("name", v))
continue
}
if cf["type"] == "dropbox" && cf["username"] == dmap["username"] {
c.String(200, `<p>The same configuration has been added</p><script>window.close()</script>`)
err := service.MyService.Storage().CheckAndMountByName(v)
if err != nil {
logger.Error("check and mount by name error: ", zap.Error(err), zap.Any("name", cf["username"]))
}
notify["status"] = "warn"
notify["message"] = "The same configuration has been added"
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
return
}
}
if len(username) > 0 {
a := strings.Split(username, "@")
username = a[0]
}
username += "_dropbox_" + strconv.FormatInt(time.Now().Unix(), 10)
dmap["client_id"] = add.AppKey
dmap["client_secret"] = add.AppSecret
dmap["token"] = `{"access_token":"` + dropbox.AccessToken + `","token_type":"bearer","refresh_token":"` + dropbox.Addition.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*3).Add(time.Minute*50).Format("15:04:05") + `.780385354Z"}`
dmap["mount_point"] = "/mnt/" + username
// data.SetValue(username, "type", "dropbox")
// data.SetValue(username, "client_id", add.AppKey)
// data.SetValue(username, "client_secret", add.AppSecret)
// data.SetValue(username, "mount_point", "/mnt/"+username)
// data.SetValue(username, "token", `{"access_token":"`+dropbox.AccessToken+`","token_type":"bearer","refresh_token":"`+dropbox.Addition.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*3).Format("15:04:05")+`.780385354Z"}`)
// e = data.Save()
// if e != nil {
// c.String(200, `<p>保存配置失败:`+e.Error()+`</p>`)
// return
// }
service.MyService.Storage().CreateConfig(dmap, username, "dropbox")
service.MyService.Storage().MountStorage("/mnt/"+username, username+":")
notify["status"] = "success"
notify["message"] = "Success"
notify["driver"] = "Dropbox"
service.MyService.Notify().SendNotify("casaos:file:recover", notify)
}
c.String(200, `<p>Just close the page</p><script>window.close()</script>`)
}

15
service/service.go
View File

@ -39,7 +39,7 @@ type Repository interface {
Rely() RelyService
Shares() SharesService
System() SystemService
Storage() StorageService
MessageBus() *message_bus.ClientWithResponses
Peer() PeerService
Other() OtherService
@ -60,9 +60,10 @@ func NewService(db *gorm.DB, RuntimePath string) Repository {
system: NewSystemService(),
health: NewHealthService(),
shares: NewSharesService(db),
storage: NewStorageService(),
other: NewOtherService(),
peer: NewPeerService(db),
other: NewOtherService(),
peer: NewPeerService(db),
}
}
@ -76,9 +77,13 @@ type store struct {
shares SharesService
connections ConnectionsService
gateway external.ManagementService
storage StorageService
health HealthService
other OtherService
}
health HealthService
other OtherService
func (c *store) Storage() StorageService {
return c.storage
}
func (c *store) Peer() PeerService {

116
service/storage.go Normal file
View File

@ -0,0 +1,116 @@
package service
import (
"io/ioutil"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/pkg/utils/file"
"github.com/IceWhaleTech/CasaOS/pkg/utils/httper"
"go.uber.org/zap"
)
type StorageService interface {
MountStorage(mountPoint, fs string) error
UnmountStorage(mountPoint string) error
GetStorages() (httper.MountList, error)
CreateConfig(data map[string]string, name string, t string) error
CheckAndMountByName(name string) error
CheckAndMountAll() error
GetConfigByName(name string) (map[string]string, error)
DeleteConfigByName(name string) error
GetConfig() (httper.RemotesResult, error)
}
type storageStruct struct {
}
func (s *storageStruct) MountStorage(mountPoint, fs string) error {
file.IsNotExistMkDir(mountPoint)
return httper.Mount(mountPoint, fs)
}
func (s *storageStruct) UnmountStorage(mountPoint string) error {
err := httper.Unmount(mountPoint)
if err == nil {
dir, _ := ioutil.ReadDir(mountPoint)
if len(dir) == 0 {
file.RMDir(mountPoint)
}
return nil
}
return err
}
func (s *storageStruct) GetStorages() (httper.MountList, error) {
return httper.GetMountList()
}
func (s *storageStruct) CreateConfig(data map[string]string, name string, t string) error {
httper.CreateConfig(data, name, t)
return nil
}
func (s *storageStruct) CheckAndMountByName(name string) error {
storages, _ := MyService.Storage().GetStorages()
currentRemote, _ := httper.GetConfigByName(name)
mountPoint := currentRemote["mount_point"]
isMount := false
for _, v := range storages.MountPoints {
if v.MountPoint == mountPoint {
isMount = true
break
}
}
if !isMount {
return MyService.Storage().MountStorage(mountPoint, name+":")
}
return nil
}
func (s *storageStruct) CheckAndMountAll() error {
storages, err := MyService.Storage().GetStorages()
if err != nil {
return err
}
logger.Info("when CheckAndMountAll storages", zap.Any("storages", storages))
section, err := httper.GetAllConfigName()
if err != nil {
return err
}
logger.Info("when CheckAndMountAll section", zap.Any("section", section))
for _, v := range section.Remotes {
currentRemote, _ := httper.GetConfigByName(v)
mountPoint := currentRemote["mount_point"]
if len(mountPoint) == 0 {
continue
}
isMount := false
for _, v := range storages.MountPoints {
if v.MountPoint == mountPoint {
isMount = true
break
}
}
if !isMount {
logger.Info("when CheckAndMountAll MountStorage", zap.String("mountPoint", mountPoint), zap.String("fs", v))
err := MyService.Storage().MountStorage(mountPoint, v+":")
if err != nil {
logger.Error("when CheckAndMountAll then", zap.String("mountPoint", mountPoint), zap.String("fs", v), zap.Error(err))
}
}
}
return nil
}
func (s *storageStruct) GetConfigByName(name string) (map[string]string, error) {
return httper.GetConfigByName(name)
}
func (s *storageStruct) DeleteConfigByName(name string) error {
return httper.DeleteConfigByName(name)
}
func (s *storageStruct) GetConfig() (httper.RemotesResult, error) {
section, err := httper.GetAllConfigName()
if err != nil {
return httper.RemotesResult{}, err
}
return section, nil
}
func NewStorageService() StorageService {
return &storageStruct{}
}

73
service/storage_old.go
View File

@ -1,73 +0,0 @@
package service
import (
"fmt"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/pkg/errors"
"gorm.io/gorm"
)
type StorageOldService interface {
CreateStorage(storage *model.Storage) error
UpdateStorage(storage *model.Storage) error
DeleteStorageById(id uint) error
GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error)
GetStorageById(id uint) (*model.Storage, error)
GetEnabledStorages() ([]model.Storage, error)
}
type storageOldStruct struct {
db *gorm.DB
}
// CreateStorage just insert storage to database
func (s *storageOldStruct) CreateStorage(storage *model.Storage) error {
return errors.WithStack(s.db.Create(storage).Error)
}
// UpdateStorage just update storage in database
func (s *storageOldStruct) UpdateStorage(storage *model.Storage) error {
return errors.WithStack(s.db.Save(storage).Error)
}
// DeleteStorageById just delete storage from database by id
func (s *storageOldStruct) DeleteStorageById(id uint) error {
return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error)
}
// GetStorages Get all storages from database order by index
func (s *storageOldStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {
storageDB := s.db.Model(&model.Storage{})
var count int64
if err := storageDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get storages count")
}
var storages []model.Storage
if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil {
return nil, 0, errors.WithStack(err)
}
return storages, count, nil
}
// GetStorageById Get Storage by id, used to update storage usually
func (s *storageOldStruct) GetStorageById(id uint) (*model.Storage, error) {
var storage model.Storage
storage.ID = id
if err := s.db.First(&storage).Error; err != nil {
return nil, errors.WithStack(err)
}
return &storage, nil
}
func (s *storageOldStruct) GetEnabledStorages() ([]model.Storage, error) {
var storages []model.Storage
if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil {
return nil, errors.WithStack(err)
}
return storages, nil
}
func NewStorageOldService(db *gorm.DB) StorageOldService {
return &storageOldStruct{db: db}
}

9
service/system.go
View File

@ -169,7 +169,7 @@ func (c *systemService) GetDirPath(path string) ([]model.Path, error) {
}
ls, err := ioutil.ReadDir(path)
ls, err := os.ReadDir(path)
if err != nil {
logger.Error("when read dir", zap.Error(err))
return []model.Path{}, err
@ -182,7 +182,12 @@ func (c *systemService) GetDirPath(path string) ([]model.Path, error) {
if err != nil {
link = filePath
}
temp := model.Path{Name: l.Name(), Path: filePath, IsDir: l.IsDir(), Date: l.ModTime(), Size: l.Size()}
tempFile, err := l.Info()
if err != nil {
logger.Error("when read dir", zap.Error(err))
return []model.Path{}, err
}
temp := model.Path{Name: l.Name(), Path: filePath, IsDir: l.IsDir(), Date: tempFile.ModTime(), Size: tempFile.Size()}
if filePath != link {
file, _ := os.Stat(link)
temp.IsDir = file.IsDir()