From 24cfa1aea29407da04f91dc71a7bb181ebaec757 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 18 Jul 2020 20:58:35 +0200 Subject: [PATCH] Backend: Refactor file indexing / skipping Signed-off-by: Michael Mayer --- internal/entity/photo.go | 4 ++-- internal/photoprism/convert.go | 4 ++-- internal/photoprism/import.go | 10 +++++----- internal/photoprism/index.go | 13 +++++++++---- internal/photoprism/index_mediafile.go | 4 ++-- internal/photoprism/purge.go | 6 +++--- internal/photoprism/purge_options.go | 4 +++- internal/photoprism/resample.go | 4 ++-- internal/query/folders.go | 16 ++++++---------- pkg/fs/done.go | 18 ++++++++++++++++++ pkg/fs/walk.go | 10 +++++----- pkg/fs/walk_test.go | 24 ++++++++++++------------ 12 files changed, 69 insertions(+), 48 deletions(-) create mode 100644 pkg/fs/done.go diff --git a/internal/entity/photo.go b/internal/entity/photo.go index 62cec1930..2ebf95029 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -211,10 +211,10 @@ func (m *Photo) Save() error { if err := UnscopedDb().Save(m).Error; err == nil { // Nothing to do. } else if !strings.Contains(strings.ToLower(err.Error()), "lock") { - log.Errorf("photo: %s (save %s)", err, m.PhotoUID) + log.Debugf("photo: %s (save %s)", err, m.PhotoUID) return err } else if err := UnscopedDb().Save(m).Error; err != nil { - log.Errorf("photo: %s (save %s after deadlock)", err, m.PhotoUID) + log.Debugf("photo: %s (save %s after deadlock)", err, m.PhotoUID) return err } diff --git a/internal/photoprism/convert.go b/internal/photoprism/convert.go index 10cf1c0f3..bbf27aa60 100644 --- a/internal/photoprism/convert.go +++ b/internal/photoprism/convert.go @@ -53,7 +53,7 @@ func (c *Convert) Start(path string) error { }() } - done := make(map[string]bool) + done := make(fs.Done) ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false) if err := ignore.Dir(path); err != nil { @@ -89,7 +89,7 @@ func (c *Convert) Start(path string) error { return nil } - done[fileName] = true + done[fileName] = fs.Processed jobs <- ConvertJob{ image: mf, diff --git a/internal/photoprism/import.go b/internal/photoprism/import.go index 30e3a0d07..99b0a5a28 100644 --- a/internal/photoprism/import.go +++ b/internal/photoprism/import.go @@ -48,9 +48,9 @@ func (imp *Import) thumbPath() string { } // Start imports media files from a directory and converts/indexes them as needed. -func (imp *Import) Start(opt ImportOptions) map[string]bool { +func (imp *Import) Start(opt ImportOptions) fs.Done { var directories []string - done := make(map[string]bool) + done := make(fs.Done) ind := imp.index importPath := opt.Path @@ -145,15 +145,15 @@ func (imp *Import) Start(opt ImportOptions) map[string]bool { var files MediaFiles for _, f := range related.Files { - if done[f.FileName()] { + if done[f.FileName()].Processed() { continue } files = append(files, f) - done[f.FileName()] = true + done[f.FileName()] = fs.Processed } - done[fileName] = true + done[fileName] = fs.Processed related.Files = files diff --git a/internal/photoprism/index.go b/internal/photoprism/index.go index 3c4340932..a0f6db238 100644 --- a/internal/photoprism/index.go +++ b/internal/photoprism/index.go @@ -55,14 +55,14 @@ func (ind *Index) Cancel() { } // Start indexes media files in the originals directory. -func (ind *Index) Start(opt IndexOptions) map[string]bool { +func (ind *Index) Start(opt IndexOptions) fs.Done { defer func() { if r := recover(); r != nil { log.Errorf("index: %s (panic)\nstack: %s", r, debug.Stack()) } }() - done := make(map[string]bool) + done := make(fs.Done) originalsPath := ind.originalsPath() optionsPath := filepath.Join(originalsPath, opt.Path) @@ -134,7 +134,7 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool { return result } - done[fileName] = true + done[fileName] = fs.Found if !fs.FileExt.Media(fileName) { return nil @@ -166,12 +166,17 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool { var files MediaFiles for _, f := range related.Files { + if done[f.FileName()].Processed() { + continue + } + files = append(files, f) filesIndexed++ - done[f.FileName()] = true + done[f.FileName()] = fs.Processed } filesIndexed++ + done[fileName] = fs.Processed related.Files = files diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index 0af1de997..8f46af9c7 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -635,7 +635,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( file.UpdatedIn = int64(time.Since(start)) if err := file.Save(); err != nil { - log.Errorf("index: %s in %s (save file)", err, logName) + log.Errorf("index: failed updating file %s", logName) result.Status = IndexFailed result.Err = err return result @@ -644,7 +644,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( file.CreatedIn = int64(time.Since(start)) if err := file.Create(); err != nil { - log.Errorf("index: %s in %s (create file)", err, logName) + log.Errorf("index: failed adding file %s", logName) result.Status = IndexFailed result.Err = err return result diff --git a/internal/photoprism/purge.go b/internal/photoprism/purge.go index a4b9504a9..f2bb90444 100644 --- a/internal/photoprism/purge.go +++ b/internal/photoprism/purge.go @@ -37,12 +37,12 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh } }() - var ignore map[string]bool + var ignore fs.Done if opt.Ignore != nil { ignore = opt.Ignore } else { - ignore = make(map[string]bool) + ignore = make(fs.Done) } purgedFiles = make(map[string]bool) @@ -76,7 +76,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh fileName := FileName(file.FileRoot, file.FileName) - if ignore[fileName] || purgedFiles[fileName] { + if ignore[fileName].Exists() || purgedFiles[fileName] { continue } diff --git a/internal/photoprism/purge_options.go b/internal/photoprism/purge_options.go index 58bd5b6a1..80e81abf2 100644 --- a/internal/photoprism/purge_options.go +++ b/internal/photoprism/purge_options.go @@ -1,8 +1,10 @@ package photoprism +import "github.com/photoprism/photoprism/pkg/fs" + type PurgeOptions struct { Path string - Ignore map[string]bool + Ignore fs.Done Dry bool Hard bool } diff --git a/internal/photoprism/resample.go b/internal/photoprism/resample.go index 6e039bae7..4af96d8e1 100644 --- a/internal/photoprism/resample.go +++ b/internal/photoprism/resample.go @@ -55,7 +55,7 @@ func (rs *Resample) Start(force bool) (err error) { }() } - done := make(map[string]bool) + done := make(fs.Done) ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false) if err := ignore.Dir(originalsPath); err != nil { @@ -91,7 +91,7 @@ func (rs *Resample) Start(force bool) (err error) { return nil } - done[fileName] = true + done[fileName] = fs.Processed relativeName := mf.RelName(originalsPath) diff --git a/internal/query/folders.go b/internal/query/folders.go index fde245576..1671c3be4 100644 --- a/internal/query/folders.go +++ b/internal/query/folders.go @@ -18,18 +18,14 @@ func FoldersByPath(rootName, rootPath, path string, recursive bool) (folders ent folders = make(entity.Folders, len(dirs)) for i, dir := range dirs { - folder := entity.FindFolder(rootName, filepath.Join(path, dir)) + newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil) - if folder == nil { - newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil) - - if err := newFolder.Create(); err != nil { - log.Errorf("folders: %s (create folder)", err.Error()) - } else { - folders[i] = newFolder - } - } else { + if err := newFolder.Create(); err == nil { + folders[i] = newFolder + } else if folder := entity.FindFolder(rootName, filepath.Join(path, dir)); folder != nil { folders[i] = *folder + } else { + log.Errorf("folders: %s (create folder)", err) } } diff --git a/pkg/fs/done.go b/pkg/fs/done.go new file mode 100644 index 000000000..bd4e666f3 --- /dev/null +++ b/pkg/fs/done.go @@ -0,0 +1,18 @@ +package fs + +type Status int8 + +const ( + Found Status = 1 + Processed Status = 2 +) + +type Done map[string]Status + +func (s Status) Exists() bool { + return s > 0 +} + +func (s Status) Processed() bool { + return s >= Processed +} diff --git a/pkg/fs/walk.go b/pkg/fs/walk.go index cd0cfe635..cfcaae462 100644 --- a/pkg/fs/walk.go +++ b/pkg/fs/walk.go @@ -6,8 +6,8 @@ import ( ) // SkipWalk returns true if the file or directory should be skipped in godirwalk.Walk() -func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, ignore *IgnoreList) (skip bool, result error) { - isDone := done[fileName] +func SkipWalk(fileName string, isDir, isSymlink bool, done Done, ignore *IgnoreList) (skip bool, result error) { + isDone := done[fileName].Exists() isIgnored := ignore.Ignore(fileName) if isSymlink { @@ -18,11 +18,11 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno if link, err := os.Stat(fileName); err == nil && link.IsDir() { resolved, err := filepath.EvalSymlinks(fileName) - if err != nil || isIgnored || isDone || done[resolved] { + if err != nil || isIgnored || isDone || done[resolved].Exists() { result = filepath.SkipDir } else { // Don't traverse symlinks if they are hidden or already done... - done[resolved] = true + done[resolved] = Found } } } else if isDir { @@ -38,7 +38,7 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno } if skip { - done[fileName] = true + done[fileName] = Found } return skip, result diff --git a/pkg/fs/walk_test.go b/pkg/fs/walk_test.go index 35577c77f..ff8e25a04 100644 --- a/pkg/fs/walk_test.go +++ b/pkg/fs/walk_test.go @@ -9,10 +9,10 @@ import ( func TestSkipWalk(t *testing.T) { t.Run("done", func(t *testing.T) { - done := make(map[string]bool) + done := make(Done) ignore := NewIgnoreList(".ppignore", true, false) - done["foo.jpg"] = true + done["foo.jpg"] = Found if skip, result := SkipWalk("testdata/directory", true, false, done, ignore); skip { assert.Nil(t, result) @@ -20,13 +20,13 @@ func TestSkipWalk(t *testing.T) { t.Fatal("should be skipped") } - assert.True(t, done["foo.jpg"]) - assert.True(t, done["testdata/directory"]) + assert.True(t, done["foo.jpg"].Exists()) + assert.True(t, done["testdata/directory"].Exists()) assert.Equal(t, 2, len(done)) }) t.Run("symlink", func(t *testing.T) { - done := make(map[string]bool) + done := make(Done) ignore := NewIgnoreList(".ppignore", true, false) if skip, result := SkipWalk("testdata/directory/subdirectory/symlink", false, true, done, ignore); skip { @@ -47,15 +47,15 @@ func TestSkipWalk(t *testing.T) { t.Fatal("should be skipped") } - assert.True(t, done["testdata/linked"]) - assert.True(t, done["testdata/directory/subdirectory/symlink"]) - assert.True(t, done["testdata/directory/subdirectory/symlink/self"]) - assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"]) + assert.True(t, done["testdata/linked"].Exists()) + assert.True(t, done["testdata/directory/subdirectory/symlink"].Exists()) + assert.True(t, done["testdata/directory/subdirectory/symlink/self"].Exists()) + assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"].Exists()) assert.Equal(t, 4, len(done)) }) t.Run("godirwalk", func(t *testing.T) { - done := make(map[string]bool) + done := make(Done) var skipped []string var skippedDirs []string testPath := "testdata" @@ -75,10 +75,10 @@ func TestSkipWalk(t *testing.T) { return result } - done[fileName] = true + done[fileName] = Found if textName := TypeText.Find(fileName, false); textName != "" { - done[textName] = true + done[textName] = Found } return nil