Backend: Refactor file indexing / skipping

Signed-off-by: Michael Mayer <michael@liquidbytes.net>

internal/entity/photo.go

@@ -211,10 +211,10 @@ func (m *Photo) Save() error {
 	if err := UnscopedDb().Save(m).Error; err == nil {
 		// Nothing to do.
 	} else if !strings.Contains(strings.ToLower(err.Error()), "lock") {
-		log.Errorf("photo: %s (save %s)", err, m.PhotoUID)
+		log.Debugf("photo: %s (save %s)", err, m.PhotoUID)
 		return err
 	} else if err := UnscopedDb().Save(m).Error; err != nil {
-		log.Errorf("photo: %s (save %s after deadlock)", err, m.PhotoUID)
+		log.Debugf("photo: %s (save %s after deadlock)", err, m.PhotoUID)
 		return err
 	}
 

internal/photoprism/convert.go

@@ -53,7 +53,7 @@ func (c *Convert) Start(path string) error {
 		}()
 	}
 
-	done := make(map[string]bool)
+	done := make(fs.Done)
 	ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
 
 	if err := ignore.Dir(path); err != nil {
@@ -89,7 +89,7 @@ func (c *Convert) Start(path string) error {
 				return nil
 			}
 
-			done[fileName] = true
+			done[fileName] = fs.Processed
 
 			jobs <- ConvertJob{
 				image:   mf,

internal/photoprism/import.go

@@ -48,9 +48,9 @@ func (imp *Import) thumbPath() string {
 }
 
 // Start imports media files from a directory and converts/indexes them as needed.
-func (imp *Import) Start(opt ImportOptions) map[string]bool {
+func (imp *Import) Start(opt ImportOptions) fs.Done {
 	var directories []string
-	done := make(map[string]bool)
+	done := make(fs.Done)
 	ind := imp.index
 	importPath := opt.Path
 
@@ -145,15 +145,15 @@ func (imp *Import) Start(opt ImportOptions) map[string]bool {
 			var files MediaFiles
 
 			for _, f := range related.Files {
-				if done[f.FileName()] {
+				if done[f.FileName()].Processed() {
 					continue
 				}
 
 				files = append(files, f)
-				done[f.FileName()] = true
+				done[f.FileName()] = fs.Processed
 			}
 
-			done[fileName] = true
+			done[fileName] = fs.Processed
 
 			related.Files = files
 

internal/photoprism/index.go

@@ -55,14 +55,14 @@ func (ind *Index) Cancel() {
 }
 
 // Start indexes media files in the originals directory.
-func (ind *Index) Start(opt IndexOptions) map[string]bool {
+func (ind *Index) Start(opt IndexOptions) fs.Done {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Errorf("index: %s (panic)\nstack: %s", r, debug.Stack())
 		}
 	}()
 
-	done := make(map[string]bool)
+	done := make(fs.Done)
 	originalsPath := ind.originalsPath()
 	optionsPath := filepath.Join(originalsPath, opt.Path)
 
@@ -134,7 +134,7 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool {
 				return result
 			}
 
-			done[fileName] = true
+			done[fileName] = fs.Found
 
 			if !fs.FileExt.Media(fileName) {
 				return nil
@@ -166,12 +166,17 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool {
 			var files MediaFiles
 
 			for _, f := range related.Files {
+				if done[f.FileName()].Processed() {
+					continue
+				}
+
 				files = append(files, f)
 				filesIndexed++
-				done[f.FileName()] = true
+				done[f.FileName()] = fs.Processed
 			}
 
 			filesIndexed++
+			done[fileName] = fs.Processed
 
 			related.Files = files
 

internal/photoprism/index_mediafile.go

@@ -635,7 +635,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 		file.UpdatedIn = int64(time.Since(start))
 
 		if err := file.Save(); err != nil {
-			log.Errorf("index: %s in %s (save file)", err, logName)
+			log.Errorf("index: failed updating file %s", logName)
 			result.Status = IndexFailed
 			result.Err = err
 			return result
@@ -644,7 +644,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 		file.CreatedIn = int64(time.Since(start))
 
 		if err := file.Create(); err != nil {
-			log.Errorf("index: %s in %s (create file)", err, logName)
+			log.Errorf("index: failed adding file %s", logName)
 			result.Status = IndexFailed
 			result.Err = err
 			return result

internal/photoprism/purge.go

@@ -37,12 +37,12 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
 		}
 	}()
 
-	var ignore map[string]bool
+	var ignore fs.Done
 
 	if opt.Ignore != nil {
 		ignore = opt.Ignore
 	} else {
-		ignore = make(map[string]bool)
+		ignore = make(fs.Done)
 	}
 
 	purgedFiles = make(map[string]bool)
@@ -76,7 +76,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
 
 			fileName := FileName(file.FileRoot, file.FileName)
 
-			if ignore[fileName] || purgedFiles[fileName] {
+			if ignore[fileName].Exists() || purgedFiles[fileName] {
 				continue
 			}
 

internal/photoprism/purge_options.go

@@ -1,8 +1,10 @@
 package photoprism
 
+import "github.com/photoprism/photoprism/pkg/fs"
+
 type PurgeOptions struct {
 	Path   string
-	Ignore map[string]bool
+	Ignore fs.Done
 	Dry    bool
 	Hard   bool
 }

internal/photoprism/resample.go

@@ -55,7 +55,7 @@ func (rs *Resample) Start(force bool) (err error) {
 		}()
 	}
 
-	done := make(map[string]bool)
+	done := make(fs.Done)
 	ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
 
 	if err := ignore.Dir(originalsPath); err != nil {
@@ -91,7 +91,7 @@ func (rs *Resample) Start(force bool) (err error) {
 				return nil
 			}
 
-			done[fileName] = true
+			done[fileName] = fs.Processed
 
 			relativeName := mf.RelName(originalsPath)
 

internal/query/folders.go

@@ -18,18 +18,14 @@ func FoldersByPath(rootName, rootPath, path string, recursive bool) (folders ent
 	folders = make(entity.Folders, len(dirs))
 
 	for i, dir := range dirs {
-		folder := entity.FindFolder(rootName, filepath.Join(path, dir))
+		newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil)
 
-		if folder == nil {
+		if err := newFolder.Create(); err == nil {
-			newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil)
+			folders[i] = newFolder
-
+		} else if folder := entity.FindFolder(rootName, filepath.Join(path, dir)); folder != nil {
-			if err := newFolder.Create(); err != nil {
-				log.Errorf("folders: %s (create folder)", err.Error())
-			} else {
-				folders[i] = newFolder
-			}
-		} else {
 			folders[i] = *folder
+		} else {
+			log.Errorf("folders: %s (create folder)", err)
 		}
 	}
 

pkg/fs/done.go

@@ -0,0 +1,18 @@
+package fs
+
+type Status int8
+
+const (
+	Found     Status = 1
+	Processed Status = 2
+)
+
+type Done map[string]Status
+
+func (s Status) Exists() bool {
+	return s > 0
+}
+
+func (s Status) Processed() bool {
+	return s >= Processed
+}

pkg/fs/walk.go

@@ -6,8 +6,8 @@ import (
 )
 
 // SkipWalk returns true if the file or directory should be skipped in godirwalk.Walk()
-func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, ignore *IgnoreList) (skip bool, result error) {
+func SkipWalk(fileName string, isDir, isSymlink bool, done Done, ignore *IgnoreList) (skip bool, result error) {
-	isDone := done[fileName]
+	isDone := done[fileName].Exists()
 	isIgnored := ignore.Ignore(fileName)
 
 	if isSymlink {
@@ -18,11 +18,11 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
 		if link, err := os.Stat(fileName); err == nil && link.IsDir() {
 			resolved, err := filepath.EvalSymlinks(fileName)
 
-			if err != nil || isIgnored || isDone || done[resolved] {
+			if err != nil || isIgnored || isDone || done[resolved].Exists() {
 				result = filepath.SkipDir
 			} else {
 				// Don't traverse symlinks if they are hidden or already done...
-				done[resolved] = true
+				done[resolved] = Found
 			}
 		}
 	} else if isDir {
@@ -38,7 +38,7 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
 	}
 
 	if skip {
-		done[fileName] = true
+		done[fileName] = Found
 	}
 
 	return skip, result

pkg/fs/walk_test.go

@@ -9,10 +9,10 @@ import (
 
 func TestSkipWalk(t *testing.T) {
 	t.Run("done", func(t *testing.T) {
-		done := make(map[string]bool)
+		done := make(Done)
 		ignore := NewIgnoreList(".ppignore", true, false)
 
-		done["foo.jpg"] = true
+		done["foo.jpg"] = Found
 
 		if skip, result := SkipWalk("testdata/directory", true, false, done, ignore); skip {
 			assert.Nil(t, result)
@@ -20,13 +20,13 @@ func TestSkipWalk(t *testing.T) {
 			t.Fatal("should be skipped")
 		}
 
-		assert.True(t, done["foo.jpg"])
+		assert.True(t, done["foo.jpg"].Exists())
-		assert.True(t, done["testdata/directory"])
+		assert.True(t, done["testdata/directory"].Exists())
 		assert.Equal(t, 2, len(done))
 	})
 
 	t.Run("symlink", func(t *testing.T) {
-		done := make(map[string]bool)
+		done := make(Done)
 		ignore := NewIgnoreList(".ppignore", true, false)
 
 		if skip, result := SkipWalk("testdata/directory/subdirectory/symlink", false, true, done, ignore); skip {
@@ -47,15 +47,15 @@ func TestSkipWalk(t *testing.T) {
 			t.Fatal("should be skipped")
 		}
 
-		assert.True(t, done["testdata/linked"])
+		assert.True(t, done["testdata/linked"].Exists())
-		assert.True(t, done["testdata/directory/subdirectory/symlink"])
+		assert.True(t, done["testdata/directory/subdirectory/symlink"].Exists())
-		assert.True(t, done["testdata/directory/subdirectory/symlink/self"])
+		assert.True(t, done["testdata/directory/subdirectory/symlink/self"].Exists())
-		assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"])
+		assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"].Exists())
 		assert.Equal(t, 4, len(done))
 	})
 
 	t.Run("godirwalk", func(t *testing.T) {
-		done := make(map[string]bool)
+		done := make(Done)
 		var skipped []string
 		var skippedDirs []string
 		testPath := "testdata"
@@ -75,10 +75,10 @@ func TestSkipWalk(t *testing.T) {
 					return result
 				}
 
-				done[fileName] = true
+				done[fileName] = Found
 
 				if textName := TypeText.Find(fileName, false); textName != "" {
-					done[textName] = true
+					done[textName] = Found
 				}
 
 				return nil