Remove fs.go
This commit is contained in:
LinkLeong
parent
10191a1be3
commit
87b190a84b
|
|
@ -1,545 +0,0 @@
|
|||
package op
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
stdpath "path"
|
||||
"time"
|
||||
|
||||
"github.com/IceWhaleTech/CasaOS/internal/driver"
|
||||
"github.com/IceWhaleTech/CasaOS/model"
|
||||
"github.com/IceWhaleTech/CasaOS/pkg/generic_sync"
|
||||
"github.com/IceWhaleTech/CasaOS/pkg/singleflight"
|
||||
"github.com/IceWhaleTech/CasaOS/pkg/utils"
|
||||
"github.com/Xhofe/go-cache"
|
||||
"github.com/pkg/errors"
|
||||
pkgerr "github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// In order to facilitate adding some other things before and after file op
|
||||
|
||||
var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
|
||||
var listG singleflight.Group[[]model.Obj]
|
||||
|
||||
func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {
|
||||
key := Key(storage, path)
|
||||
objs, ok := listCache.Get(key)
|
||||
if ok {
|
||||
for i, obj := range objs {
|
||||
if obj.GetName() == oldObj.GetName() {
|
||||
objs[i] = newObj
|
||||
break
|
||||
}
|
||||
}
|
||||
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
|
||||
}
|
||||
}
|
||||
|
||||
func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
|
||||
key := Key(storage, path)
|
||||
objs, ok := listCache.Get(key)
|
||||
if ok {
|
||||
for i, oldObj := range objs {
|
||||
if oldObj.GetName() == obj.GetName() {
|
||||
objs = append(objs[:i], objs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
|
||||
}
|
||||
}
|
||||
|
||||
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
|
||||
|
||||
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
|
||||
key := Key(storage, path)
|
||||
objs, ok := listCache.Get(key)
|
||||
if ok {
|
||||
for i, obj := range objs {
|
||||
if obj.GetName() == newObj.GetName() {
|
||||
objs[i] = newObj
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Simple separation of files and folders
|
||||
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
|
||||
objs = append(objs, newObj)
|
||||
} else {
|
||||
objs = append([]model.Obj{newObj}, objs...)
|
||||
}
|
||||
|
||||
if storage.Config().LocalSort {
|
||||
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
|
||||
log.Debug("addCacheObj: wait start sort")
|
||||
debounce(func() {
|
||||
log.Debug("addCacheObj: start sort")
|
||||
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
|
||||
addSortDebounceMap.Delete(key)
|
||||
})
|
||||
}
|
||||
|
||||
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
|
||||
}
|
||||
}
|
||||
|
||||
func ClearCache(storage driver.Driver, path string) {
|
||||
listCache.Del(Key(storage, path))
|
||||
}
|
||||
|
||||
func Key(storage driver.Driver, path string) string {
|
||||
return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))
|
||||
}
|
||||
|
||||
// List files in storage, not contains virtual file
|
||||
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
path = utils.FixAndCleanPath(path)
|
||||
log.Debugf("op.List %s", path)
|
||||
key := Key(storage, path)
|
||||
if !utils.IsBool(refresh...) {
|
||||
if files, ok := listCache.Get(key); ok {
|
||||
log.Debugf("use cache when list %s", path)
|
||||
return files, nil
|
||||
}
|
||||
}
|
||||
dir, err := GetUnwrap(ctx, storage, path)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get dir")
|
||||
}
|
||||
log.Debugf("list dir: %+v", dir)
|
||||
if !dir.IsDir() {
|
||||
return nil, errors.WithStack(errors.New("not a folder"))
|
||||
}
|
||||
objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
|
||||
files, err := storage.List(ctx, dir, args)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to list objs")
|
||||
}
|
||||
// set path
|
||||
for _, f := range files {
|
||||
if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" {
|
||||
s.SetPath(stdpath.Join(dir.GetPath(), f.GetName()))
|
||||
}
|
||||
}
|
||||
// warp obj name
|
||||
model.WrapObjsName(files)
|
||||
// call hooks
|
||||
go func(reqPath string, files []model.Obj) {
|
||||
for _, hook := range ObjsUpdateHooks {
|
||||
hook(args.ReqPath, files)
|
||||
}
|
||||
}(args.ReqPath, files)
|
||||
|
||||
// sort objs
|
||||
if storage.Config().LocalSort {
|
||||
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
|
||||
}
|
||||
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
|
||||
|
||||
if !storage.Config().NoCache {
|
||||
if len(files) > 0 {
|
||||
log.Debugf("set cache: %s => %+v", key, files)
|
||||
listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
|
||||
} else {
|
||||
log.Debugf("del cache: %s", key)
|
||||
listCache.Del(key)
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
})
|
||||
return objs, err
|
||||
}
|
||||
|
||||
// Get object from list of files
|
||||
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
|
||||
path = utils.FixAndCleanPath(path)
|
||||
log.Debugf("op.Get %s", path)
|
||||
|
||||
// is root folder
|
||||
if utils.PathEqual(path, "/") {
|
||||
var rootObj model.Obj
|
||||
switch r := storage.GetAddition().(type) {
|
||||
case driver.IRootId:
|
||||
rootObj = &model.Object{
|
||||
ID: r.GetRootId(),
|
||||
Name: RootName,
|
||||
Size: 0,
|
||||
Modified: storage.GetStorage().Modified,
|
||||
IsFolder: true,
|
||||
Path: path,
|
||||
}
|
||||
case driver.IRootPath:
|
||||
rootObj = &model.Object{
|
||||
Path: r.GetRootPath(),
|
||||
Name: RootName,
|
||||
Size: 0,
|
||||
Modified: storage.GetStorage().Modified,
|
||||
IsFolder: true,
|
||||
}
|
||||
default:
|
||||
if storage, ok := storage.(driver.Getter); ok {
|
||||
obj, err := storage.GetRoot(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get root obj")
|
||||
}
|
||||
rootObj = obj
|
||||
}
|
||||
}
|
||||
if rootObj == nil {
|
||||
return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method")
|
||||
}
|
||||
return &model.ObjWrapName{
|
||||
Name: RootName,
|
||||
Obj: rootObj,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// not root folder
|
||||
dir, name := stdpath.Split(path)
|
||||
files, err := List(ctx, storage, dir, model.ListArgs{})
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get parent list")
|
||||
}
|
||||
for _, f := range files {
|
||||
// TODO maybe copy obj here
|
||||
if f.GetName() == name {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
log.Debugf("cant find obj with name: %s", name)
|
||||
return nil, errors.WithStack(errors.New("object not found"))
|
||||
}
|
||||
|
||||
func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
|
||||
obj, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return model.UnwrapObjs(obj), err
|
||||
}
|
||||
|
||||
var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
|
||||
var linkG singleflight.Group[*model.Link]
|
||||
|
||||
// Link get link, if is an url. should have an expiry time
|
||||
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
file, err := GetUnwrap(ctx, storage, path)
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithMessage(err, "failed to get file")
|
||||
}
|
||||
if file.IsDir() {
|
||||
return nil, nil, errors.WithStack(errors.New("not a file"))
|
||||
}
|
||||
key := Key(storage, path) + ":" + args.IP
|
||||
if link, ok := linkCache.Get(key); ok {
|
||||
return link, file, nil
|
||||
}
|
||||
fn := func() (*model.Link, error) {
|
||||
link, err := storage.Link(ctx, file, args)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed get link")
|
||||
}
|
||||
if link.Expiration != nil {
|
||||
linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))
|
||||
}
|
||||
return link, nil
|
||||
}
|
||||
link, err, _ := linkG.Do(key, fn)
|
||||
return link, file, err
|
||||
}
|
||||
|
||||
// Other api
|
||||
func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) {
|
||||
obj, err := GetUnwrap(ctx, storage, args.Path)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessagef(err, "failed to get obj")
|
||||
}
|
||||
if o, ok := storage.(driver.Other); ok {
|
||||
return o.Other(ctx, model.OtherArgs{
|
||||
Obj: obj,
|
||||
Method: args.Method,
|
||||
Data: args.Data,
|
||||
})
|
||||
} else {
|
||||
return nil, errors.New("not implement")
|
||||
}
|
||||
}
|
||||
|
||||
var mkdirG singleflight.Group[interface{}]
|
||||
|
||||
func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
path = utils.FixAndCleanPath(path)
|
||||
key := Key(storage, path)
|
||||
_, err, _ := mkdirG.Do(key, func() (interface{}, error) {
|
||||
// check if dir exists
|
||||
f, err := GetUnwrap(ctx, storage, path)
|
||||
if err != nil {
|
||||
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
|
||||
parentPath, dirName := stdpath.Split(path)
|
||||
err = MakeDir(ctx, storage, parentPath)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
|
||||
}
|
||||
parentDir, err := GetUnwrap(ctx, storage, parentPath)
|
||||
// this should not happen
|
||||
if err != nil {
|
||||
return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
|
||||
}
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.MkdirResult:
|
||||
var newObj model.Obj
|
||||
newObj, err = s.MakeDir(ctx, parentDir, dirName)
|
||||
if err == nil {
|
||||
if newObj != nil {
|
||||
addCacheObj(storage, parentPath, model.WrapObjName(newObj))
|
||||
} else if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, parentPath)
|
||||
}
|
||||
}
|
||||
case driver.Mkdir:
|
||||
err = s.MakeDir(ctx, parentDir, dirName)
|
||||
if err == nil && !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, parentPath)
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("not implement")
|
||||
}
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
return nil, errors.WithMessage(err, "failed to check if dir exists")
|
||||
}
|
||||
// dir exists
|
||||
if f.IsDir() {
|
||||
return nil, nil
|
||||
}
|
||||
// dir to make is a file
|
||||
return nil, errors.New("file exists")
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
srcPath = utils.FixAndCleanPath(srcPath)
|
||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
srcObj := model.UnwrapObjs(srcRawObj)
|
||||
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get dst dir")
|
||||
}
|
||||
srcDirPath := stdpath.Dir(srcPath)
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.MoveResult:
|
||||
var newObj model.Obj
|
||||
newObj, err = s.Move(ctx, srcObj, dstDir)
|
||||
if err == nil {
|
||||
delCacheObj(storage, srcDirPath, srcRawObj)
|
||||
if newObj != nil {
|
||||
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
|
||||
} else if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
}
|
||||
case driver.Move:
|
||||
err = s.Move(ctx, srcObj, dstDir)
|
||||
if err == nil {
|
||||
delCacheObj(storage, srcDirPath, srcRawObj)
|
||||
if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return errors.New("not implement")
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
srcPath = utils.FixAndCleanPath(srcPath)
|
||||
srcRawObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
srcObj := model.UnwrapObjs(srcRawObj)
|
||||
srcDirPath := stdpath.Dir(srcPath)
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.RenameResult:
|
||||
var newObj model.Obj
|
||||
newObj, err = s.Rename(ctx, srcObj, dstName)
|
||||
if err == nil {
|
||||
if newObj != nil {
|
||||
updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
|
||||
} else if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, srcDirPath)
|
||||
}
|
||||
}
|
||||
case driver.Rename:
|
||||
err = s.Rename(ctx, srcObj, dstName)
|
||||
if err == nil && !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, srcDirPath)
|
||||
}
|
||||
default:
|
||||
return errors.New("not implement")
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
// Copy Just copy file[s] in a storage
|
||||
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
srcPath = utils.FixAndCleanPath(srcPath)
|
||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||
srcObj, err := GetUnwrap(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get dst dir")
|
||||
}
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.CopyResult:
|
||||
var newObj model.Obj
|
||||
newObj, err = s.Copy(ctx, srcObj, dstDir)
|
||||
if err == nil {
|
||||
if newObj != nil {
|
||||
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
|
||||
} else if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
}
|
||||
case driver.Copy:
|
||||
err = s.Copy(ctx, srcObj, dstDir)
|
||||
if err == nil && !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
default:
|
||||
return errors.New("not implement")
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
func Remove(ctx context.Context, storage driver.Driver, path string) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
path = utils.FixAndCleanPath(path)
|
||||
rawObj, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
// if object not found, it's ok
|
||||
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
|
||||
return nil
|
||||
}
|
||||
return errors.WithMessage(err, "failed to get object")
|
||||
}
|
||||
dirPath := stdpath.Dir(path)
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.Remove:
|
||||
err = s.Remove(ctx, model.UnwrapObjs(rawObj))
|
||||
if err == nil {
|
||||
delCacheObj(storage, dirPath, rawObj)
|
||||
}
|
||||
default:
|
||||
return errors.New("not implement")
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error {
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
|
||||
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
|
||||
}
|
||||
defer func() {
|
||||
if f, ok := file.GetReadCloser().(*os.File); ok {
|
||||
err := os.RemoveAll(f.Name())
|
||||
if err != nil {
|
||||
log.Errorf("failed to remove file [%s]", f.Name())
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
log.Errorf("failed to close file streamer, %v", err)
|
||||
}
|
||||
}()
|
||||
// if file exist and size = 0, delete it
|
||||
dstDirPath = utils.FixAndCleanPath(dstDirPath)
|
||||
dstPath := stdpath.Join(dstDirPath, file.GetName())
|
||||
fi, err := GetUnwrap(ctx, storage, dstPath)
|
||||
if err == nil {
|
||||
if fi.GetSize() == 0 {
|
||||
err = Remove(ctx, storage, dstPath)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed remove file that exist and have size 0")
|
||||
}
|
||||
} else {
|
||||
file.Old = fi
|
||||
}
|
||||
}
|
||||
err = MakeDir(ctx, storage, dstDirPath)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
|
||||
}
|
||||
parentDir, err := GetUnwrap(ctx, storage, dstDirPath)
|
||||
// this should not happen
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
|
||||
}
|
||||
// if up is nil, set a default to prevent panic
|
||||
if up == nil {
|
||||
up = func(p int) {}
|
||||
}
|
||||
|
||||
switch s := storage.(type) {
|
||||
case driver.PutResult:
|
||||
var newObj model.Obj
|
||||
newObj, err = s.Put(ctx, parentDir, file, up)
|
||||
if err == nil {
|
||||
if newObj != nil {
|
||||
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
|
||||
} else if !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
}
|
||||
case driver.Put:
|
||||
err = s.Put(ctx, parentDir, file, up)
|
||||
if err == nil && !utils.IsBool(lazyCache...) {
|
||||
ClearCache(storage, dstDirPath)
|
||||
}
|
||||
default:
|
||||
return errors.New("not implement")
|
||||
}
|
||||
log.Debugf("put file [%s] done", file.GetName())
|
||||
//if err == nil {
|
||||
// //clear cache
|
||||
// key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
|
||||
// listCache.Del(key)
|
||||
//}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
Loading…
Reference in New Issue