Remove fs.go

This commit is contained in:
LinkLeong 2023-03-17 08:00:06 +00:00
parent 10191a1be3
commit 87b190a84b

View file

@ -1,545 +0,0 @@
package op
import (
"context"
"os"
stdpath "path"
"time"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/generic_sync"
"github.com/IceWhaleTech/CasaOS/pkg/singleflight"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/Xhofe/go-cache"
"github.com/pkg/errors"
pkgerr "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// In order to facilitate adding some other things before and after file op
var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var listG singleflight.Group[[]model.Obj]
func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == oldObj.GetName() {
objs[i] = newObj
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, oldObj := range objs {
if oldObj.GetName() == obj.GetName() {
objs = append(objs[:i], objs[i+1:]...)
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == newObj.GetName() {
objs[i] = newObj
return
}
}
// Simple separation of files and folders
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
objs = append(objs, newObj)
} else {
objs = append([]model.Obj{newObj}, objs...)
}
if storage.Config().LocalSort {
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
log.Debug("addCacheObj: wait start sort")
debounce(func() {
log.Debug("addCacheObj: start sort")
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
addSortDebounceMap.Delete(key)
})
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func ClearCache(storage driver.Driver, path string) {
listCache.Del(Key(storage, path))
}
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))
}
// List files in storage, not contains virtual file
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
log.Debugf("op.List %s", path)
key := Key(storage, path)
if !utils.IsBool(refresh...) {
if files, ok := listCache.Get(key); ok {
log.Debugf("use cache when list %s", path)
return files, nil
}
}
dir, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
}
log.Debugf("list dir: %+v", dir)
if !dir.IsDir() {
return nil, errors.WithStack(errors.New("not a folder"))
}
objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
files, err := storage.List(ctx, dir, args)
if err != nil {
return nil, errors.Wrapf(err, "failed to list objs")
}
// set path
for _, f := range files {
if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" {
s.SetPath(stdpath.Join(dir.GetPath(), f.GetName()))
}
}
// warp obj name
model.WrapObjsName(files)
// call hooks
go func(reqPath string, files []model.Obj) {
for _, hook := range ObjsUpdateHooks {
hook(args.ReqPath, files)
}
}(args.ReqPath, files)
// sort objs
if storage.Config().LocalSort {
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
} else {
log.Debugf("del cache: %s", key)
listCache.Del(key)
}
}
return files, nil
})
return objs, err
}
// Get object from list of files
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
path = utils.FixAndCleanPath(path)
log.Debugf("op.Get %s", path)
// is root folder
if utils.PathEqual(path, "/") {
var rootObj model.Obj
switch r := storage.GetAddition().(type) {
case driver.IRootId:
rootObj = &model.Object{
ID: r.GetRootId(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
Path: path,
}
case driver.IRootPath:
rootObj = &model.Object{
Path: r.GetRootPath(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
}
default:
if storage, ok := storage.(driver.Getter); ok {
obj, err := storage.GetRoot(ctx)
if err != nil {
return nil, errors.WithMessage(err, "failed get root obj")
}
rootObj = obj
}
}
if rootObj == nil {
return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method")
}
return &model.ObjWrapName{
Name: RootName,
Obj: rootObj,
}, nil
}
// not root folder
dir, name := stdpath.Split(path)
files, err := List(ctx, storage, dir, model.ListArgs{})
if err != nil {
return nil, errors.WithMessage(err, "failed get parent list")
}
for _, f := range files {
// TODO maybe copy obj here
if f.GetName() == name {
return f, nil
}
}
log.Debugf("cant find obj with name: %s", name)
return nil, errors.WithStack(errors.New("object not found"))
}
func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
obj, err := Get(ctx, storage, path)
if err != nil {
return nil, err
}
return model.UnwrapObjs(obj), err
}
var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
var linkG singleflight.Group[*model.Link]
// Link get link, if is an url. should have an expiry time
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
file, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed to get file")
}
if file.IsDir() {
return nil, nil, errors.WithStack(errors.New("not a file"))
}
key := Key(storage, path) + ":" + args.IP
if link, ok := linkCache.Get(key); ok {
return link, file, nil
}
fn := func() (*model.Link, error) {
link, err := storage.Link(ctx, file, args)
if err != nil {
return nil, errors.Wrapf(err, "failed get link")
}
if link.Expiration != nil {
linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))
}
return link, nil
}
link, err, _ := linkG.Do(key, fn)
return link, file, err
}
// Other api
func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) {
obj, err := GetUnwrap(ctx, storage, args.Path)
if err != nil {
return nil, errors.WithMessagef(err, "failed to get obj")
}
if o, ok := storage.(driver.Other); ok {
return o.Other(ctx, model.OtherArgs{
Obj: obj,
Method: args.Method,
Data: args.Data,
})
} else {
return nil, errors.New("not implement")
}
}
var mkdirG singleflight.Group[interface{}]
func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
key := Key(storage, path)
_, err, _ := mkdirG.Do(key, func() (interface{}, error) {
// check if dir exists
f, err := GetUnwrap(ctx, storage, path)
if err != nil {
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
parentPath, dirName := stdpath.Split(path)
err = MakeDir(ctx, storage, parentPath)
if err != nil {
return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
parentDir, err := GetUnwrap(ctx, storage, parentPath)
// this should not happen
if err != nil {
return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
switch s := storage.(type) {
case driver.MkdirResult:
var newObj model.Obj
newObj, err = s.MakeDir(ctx, parentDir, dirName)
if err == nil {
if newObj != nil {
addCacheObj(storage, parentPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
}
case driver.Mkdir:
err = s.MakeDir(ctx, parentDir, dirName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
default:
return nil, errors.New("not implement")
}
return nil, errors.WithStack(err)
}
return nil, errors.WithMessage(err, "failed to check if dir exists")
}
// dir exists
if f.IsDir() {
return nil, nil
}
// dir to make is a file
return nil, errors.New("file exists")
})
return err
}
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.MoveResult:
var newObj model.Obj
newObj, err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Move:
err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.RenameResult:
var newObj model.Obj
newObj, err = s.Rename(ctx, srcObj, dstName)
if err == nil {
if newObj != nil {
updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
}
case driver.Rename:
err = s.Rename(ctx, srcObj, dstName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
// Copy Just copy file[s] in a storage
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcObj, err := GetUnwrap(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
switch s := storage.(type) {
case driver.CopyResult:
var newObj model.Obj
newObj, err = s.Copy(ctx, srcObj, dstDir)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Copy:
err = s.Copy(ctx, srcObj, dstDir)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Remove(ctx context.Context, storage driver.Driver, path string) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
rawObj, err := Get(ctx, storage, path)
if err != nil {
// if object not found, it's ok
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
return nil
}
return errors.WithMessage(err, "failed to get object")
}
dirPath := stdpath.Dir(path)
switch s := storage.(type) {
case driver.Remove:
err = s.Remove(ctx, model.UnwrapObjs(rawObj))
if err == nil {
delCacheObj(storage, dirPath, rawObj)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
defer func() {
if f, ok := file.GetReadCloser().(*os.File); ok {
err := os.RemoveAll(f.Name())
if err != nil {
log.Errorf("failed to remove file [%s]", f.Name())
}
}
}()
defer func() {
if err := file.Close(); err != nil {
log.Errorf("failed to close file streamer, %v", err)
}
}()
// if file exist and size = 0, delete it
dstDirPath = utils.FixAndCleanPath(dstDirPath)
dstPath := stdpath.Join(dstDirPath, file.GetName())
fi, err := GetUnwrap(ctx, storage, dstPath)
if err == nil {
if fi.GetSize() == 0 {
err = Remove(ctx, storage, dstPath)
if err != nil {
return errors.WithMessagef(err, "failed remove file that exist and have size 0")
}
} else {
file.Old = fi
}
}
err = MakeDir(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
}
parentDir, err := GetUnwrap(ctx, storage, dstDirPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
}
// if up is nil, set a default to prevent panic
if up == nil {
up = func(p int) {}
}
switch s := storage.(type) {
case driver.PutResult:
var newObj model.Obj
newObj, err = s.Put(ctx, parentDir, file, up)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Put:
err = s.Put(ctx, parentDir, file, up)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
log.Debugf("put file [%s] done", file.GetName())
//if err == nil {
// //clear cache
// key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
// listCache.Del(key)
//}
return errors.WithStack(err)
}