diff --git a/go.mod b/go.mod index 351ca7f94..1772a4bf1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/guregu/null v3.4.0+incompatible // indirect github.com/jinzhu/gorm v1.9.5 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/karrick/godirwalk v1.15.6 // indirect + github.com/karrick/godirwalk v1.15.6 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/leandro-lugaresi/hub v1.1.0 diff --git a/internal/config/db.go b/internal/config/db.go index 10b92f2b4..4e65d6fd4 100644 --- a/internal/config/db.go +++ b/internal/config/db.go @@ -100,7 +100,7 @@ func (c *Config) connectToDatabase(ctx context.Context) error { go tidb.Start(ctx, c.TidbServerPath(), c.TidbServerPort(), c.TidbServerHost(), c.Debug()) - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) } for i := 1; i <= 12; i++ { diff --git a/internal/photoprism/convert.go b/internal/photoprism/convert.go index 9eaece8b6..844b022a4 100644 --- a/internal/photoprism/convert.go +++ b/internal/photoprism/convert.go @@ -4,12 +4,12 @@ import ( "bytes" "errors" "fmt" - "os" "os/exec" "path/filepath" "runtime" "sync" + "github.com/karrick/godirwalk" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/mutex" @@ -49,37 +49,39 @@ func (c *Convert) Start(path string) error { }() } - err := filepath.Walk(path, func(fileName string, fileInfo os.FileInfo, err error) error { - defer func() { - if err := recover(); err != nil { - log.Errorf("convert: %s [panic]", err) + done := make(map[string]bool) + + err := godirwalk.Walk(path, &godirwalk.Options{ + Callback: func(fileName string, info *godirwalk.Dirent) error { + defer func() { + if err := recover(); err != nil { + log.Errorf("convert: %s [panic]", err) + } + }() + + if mutex.Worker.Canceled() { + return errors.New("convert: canceled") } - }() - if mutex.Worker.Canceled() { - return errors.New("convert: canceled") - } + if skip, result := fs.SkipGodirwalk(fileName, info, done); skip { + return result + } + + mf, err := NewMediaFile(fileName) + + if err != nil || !(mf.IsRaw() || mf.IsHEIF() || mf.IsImageOther()) { + return nil + } + + jobs <- ConvertJob{ + image: mf, + convert: c, + } - if err != nil { return nil - } - - if fileInfo.IsDir() { - return nil - } - - mf, err := NewMediaFile(fileName) - - if err != nil || !(mf.IsRaw() || mf.IsHEIF() || mf.IsImageOther()) { - return nil - } - - jobs <- ConvertJob{ - image: mf, - convert: c, - } - - return nil + }, + Unsorted: true, + FollowSymbolicLinks: true, }) close(jobs) diff --git a/internal/photoprism/import.go b/internal/photoprism/import.go index ec87c54db..69c94dcb4 100644 --- a/internal/photoprism/import.go +++ b/internal/photoprism/import.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/karrick/godirwalk" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/event" @@ -79,81 +80,85 @@ func (imp *Import) Start(opt ImportOptions) { indexOpt := IndexOptionsAll() - err := filepath.Walk(importPath, func(fileName string, fileInfo os.FileInfo, err error) error { - defer func() { - if err := recover(); err != nil { - log.Errorf("import: %s [panic]", err) - } - }() + err := godirwalk.Walk(importPath, &godirwalk.Options{ + Callback: func(fileName string, info *godirwalk.Dirent) error { + defer func() { + if err := recover(); err != nil { + log.Errorf("import: %s [panic]", err) + } + }() - if mutex.Worker.Canceled() { - return errors.New("import canceled") - } - - if err != nil || done[fileName] { - return nil - } - - if fileInfo.IsDir() { - if fileName != importPath { - directories = append(directories, fileName) + if mutex.Worker.Canceled() { + return errors.New("import canceled") } - return nil - } - - if strings.HasPrefix(filepath.Base(fileName), ".") { - done[fileName] = true - - if !opt.RemoveDotFiles { + if done[fileName] { return nil } - if err := os.Remove(fileName); err != nil { - log.Errorf("import: could not remove \"%s\" (%s)", fileName, err.Error()) + if info.IsDir() { + if fileName != importPath { + directories = append(directories, fileName) + } + } else if info.IsRegular() && strings.HasPrefix(filepath.Base(fileName), ".") { + done[fileName] = true + + if !opt.RemoveDotFiles { + return nil + } + + if err := os.Remove(fileName); err != nil { + log.Errorf("import: could not remove \"%s\" (%s)", fileName, err.Error()) + } + + return nil + } + + if skip, result := fs.SkipGodirwalk(fileName, info, done); skip { + return result + } + + mf, err := NewMediaFile(fileName) + + if err != nil || !mf.IsPhoto() { + return nil + } + + related, err := mf.RelatedFiles(imp.conf.Settings().Library.GroupRelated) + + if err != nil { + event.Error(fmt.Sprintf("import: %s", err.Error())) + + return nil + } + + var files MediaFiles + + for _, f := range related.Files { + if done[f.FileName()] { + continue + } + + files = append(files, f) + done[f.FileName()] = true + } + + done[mf.FileName()] = true + + related.Files = files + + jobs <- ImportJob{ + FileName: fileName, + Related: related, + IndexOpt: indexOpt, + ImportOpt: opt, + Imp: imp, } return nil - } - - mf, err := NewMediaFile(fileName) - - if err != nil || !mf.IsPhoto() { - return nil - } - - related, err := mf.RelatedFiles(imp.conf.Settings().Library.GroupRelated) - - if err != nil { - event.Error(fmt.Sprintf("import: %s", err.Error())) - - return nil - } - - var files MediaFiles - - for _, f := range related.Files { - if done[f.FileName()] { - continue - } - - files = append(files, f) - done[f.FileName()] = true - } - - done[mf.FileName()] = true - - related.Files = files - - jobs <- ImportJob{ - FileName: fileName, - Related: related, - IndexOpt: indexOpt, - ImportOpt: opt, - Imp: imp, - } - - return nil + }, + Unsorted: false, + FollowSymbolicLinks: true, }) close(jobs) diff --git a/internal/photoprism/index.go b/internal/photoprism/index.go index a1a9261be..04c7c8f47 100644 --- a/internal/photoprism/index.go +++ b/internal/photoprism/index.go @@ -3,12 +3,10 @@ package photoprism import ( "errors" "fmt" - "os" - "path/filepath" - "strings" "sync" "github.com/jinzhu/gorm" + "github.com/karrick/godirwalk" "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/event" @@ -89,68 +87,62 @@ func (ind *Index) Start(options IndexOptions) map[string]bool { }() } - err := filepath.Walk(originalsPath, func(fileName string, fileInfo os.FileInfo, err error) error { - defer func() { - if err := recover(); err != nil { - log.Errorf("index: %s [panic]", err) - } - }() + err := godirwalk.Walk(originalsPath, &godirwalk.Options{ + Callback: func(fileName string, info *godirwalk.Dirent) error { + defer func() { + if err := recover(); err != nil { + log.Errorf("index: %s [panic]", err) + } + }() - if mutex.Worker.Canceled() { - return errors.New("indexing canceled") - } - - if err != nil || done[fileName] { - return nil - } - - hidden := strings.HasPrefix(filepath.Base(fileName), ".") - - if fileInfo.IsDir() && hidden { - return filepath.SkipDir - } - - if fileInfo.IsDir() || hidden { - return nil - } - - mf, err := NewMediaFile(fileName) - - if err != nil || !mf.IsPhoto() { - return nil - } - - related, err := mf.RelatedFiles(ind.conf.Settings().Library.GroupRelated) - - if err != nil { - log.Warnf("index: %s", err.Error()) - - return nil - } - - var files MediaFiles - - for _, f := range related.Files { - if done[f.FileName()] { - continue + if mutex.Worker.Canceled() { + return errors.New("indexing canceled") } - files = append(files, f) - done[f.FileName()] = true - } + if skip, result := fs.SkipGodirwalk(fileName, info, done); skip { + return result + } - done[mf.FileName()] = true + mf, err := NewMediaFile(fileName) - related.Files = files + if err != nil || !mf.IsPhoto() { + return nil + } - jobs <- IndexJob{ - FileName: mf.FileName(), - Related: related, - IndexOpt: options, - Ind: ind, - } + related, err := mf.RelatedFiles(ind.conf.Settings().Library.GroupRelated) - return nil + if err != nil { + log.Warnf("index: %s", err.Error()) + + return nil + } + + var files MediaFiles + + for _, f := range related.Files { + if done[f.FileName()] { + continue + } + + files = append(files, f) + done[f.FileName()] = true + } + + done[mf.FileName()] = true + + related.Files = files + + jobs <- IndexJob{ + FileName: mf.FileName(), + Related: related, + IndexOpt: options, + Ind: ind, + } + + return nil + }, + Unsorted: false, + FollowSymbolicLinks: true, }) close(jobs) diff --git a/internal/photoprism/resample.go b/internal/photoprism/resample.go index 4642fb45e..58fe1e629 100644 --- a/internal/photoprism/resample.go +++ b/internal/photoprism/resample.go @@ -2,14 +2,14 @@ package photoprism import ( "errors" - "os" "path/filepath" - "strings" "sync" + "github.com/karrick/godirwalk" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/mutex" + "github.com/photoprism/photoprism/pkg/fs" ) // Resample represents a thumbnail generator. @@ -46,42 +46,48 @@ func (rs *Resample) Start(force bool) error { }() } - err := filepath.Walk(originalsPath, func(filename string, fileInfo os.FileInfo, err error) error { - defer func() { - if err := recover(); err != nil { - log.Errorf("resample: %s [panic]", err) + done := make(map[string]bool) + + err := godirwalk.Walk(originalsPath, &godirwalk.Options{ + Callback: func(fileName string, info *godirwalk.Dirent) error { + defer func() { + if err := recover(); err != nil { + log.Errorf("resample: %s [panic]", err) + } + }() + + if mutex.Worker.Canceled() { + return errors.New("resample: canceled") } - }() - if mutex.Worker.Canceled() { - return errors.New("resample: canceled") - } + if skip, result := fs.SkipGodirwalk(fileName, info, done); skip { + return result + } + + mf, err := NewMediaFile(fileName) + + if err != nil || !mf.IsJpeg() { + return nil + } + + relativeName := mf.RelativeName(originalsPath) + + event.Publish("index.thumbnails", event.Data{ + "fileName": relativeName, + "baseName": filepath.Base(relativeName), + "force": force, + }) + + jobs <- ResampleJob{ + mediaFile: mf, + path: thumbnailsPath, + force: force, + } - if err != nil || fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") { return nil - } - - mf, err := NewMediaFile(filename) - - if err != nil || !mf.IsJpeg() { - return nil - } - - fileName := mf.RelativeName(originalsPath) - - event.Publish("index.thumbnails", event.Data{ - "fileName": fileName, - "baseName": filepath.Base(fileName), - "force": force, - }) - - jobs <- ResampleJob{ - mediaFile: mf, - path: thumbnailsPath, - force: force, - } - - return nil + }, + Unsorted: true, + FollowSymbolicLinks: true, }) close(jobs) diff --git a/pkg/fs/walk.go b/pkg/fs/walk.go new file mode 100644 index 000000000..ac1f4ba62 --- /dev/null +++ b/pkg/fs/walk.go @@ -0,0 +1,42 @@ +package fs + +import ( + "os" + "path/filepath" + "strings" + + "github.com/karrick/godirwalk" +) + +// SkipGodirwalk returns true if the file or directory should be skipped in godirwalk.Walk() +func SkipGodirwalk(fileName string, info *godirwalk.Dirent, done map[string]bool) (skip bool, result error) { + isDone := done[fileName] + isHidden := strings.HasPrefix(filepath.Base(fileName), ".") + isDir := info.IsDir() + isSymlink := info.IsSymlink() + + done[fileName] = true + + if isSymlink { + skip = true + + // Symlink points to directory? + if link, err := os.Stat(fileName); err == nil && link.IsDir() && (isHidden || isDone || done[link.Name()]) { + // Don't traverse symlinks if they are hidden or already done... + done[link.Name()] = true + result = filepath.SkipDir + } + } else if isDir { + skip = true + + if isHidden || isDone { + // Don't traverse directories if they are hidden or already done... + result = filepath.SkipDir + } + } else if isHidden || isDone { + // Skip files that are hidden or already done... + skip = true + } + + return skip, result +}