Backend: Replace filepath.Walk with godirwalk.Walk #267

Signed-off-by: Michael Mayer <michael@liquidbytes.net>

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/guregu/null v3.4.0+incompatible // indirect
 	github.com/jinzhu/gorm v1.9.5
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/karrick/godirwalk v1.15.6 // indirect
+	github.com/karrick/godirwalk v1.15.6
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/leandro-lugaresi/hub v1.1.0

internal/config/db.go

@@ -100,7 +100,7 @@ func (c *Config) connectToDatabase(ctx context.Context) error {
 
 			go tidb.Start(ctx, c.TidbServerPath(), c.TidbServerPort(), c.TidbServerHost(), c.Debug())
 
-			time.Sleep(2 * time.Second)
+			time.Sleep(5 * time.Second)
 		}
 
 		for i := 1; i <= 12; i++ {

internal/photoprism/convert.go

@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sync"
 
+	"github.com/karrick/godirwalk"
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/mutex"
@@ -49,37 +49,39 @@ func (c *Convert) Start(path string) error {
 		}()
 	}
 
-	err := filepath.Walk(path, func(fileName string, fileInfo os.FileInfo, err error) error {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Errorf("convert: %s [panic]", err)
+	done := make(map[string]bool)
+
+	err := godirwalk.Walk(path, &godirwalk.Options{
+		Callback: func(fileName string, info *godirwalk.Dirent) error {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Errorf("convert: %s [panic]", err)
+				}
+			}()
+
+			if mutex.Worker.Canceled() {
+				return errors.New("convert: canceled")
 			}
-		}()
 
-		if mutex.Worker.Canceled() {
-			return errors.New("convert: canceled")
-		}
+			if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
+				return result
+			}
+
+			mf, err := NewMediaFile(fileName)
+
+			if err != nil || !(mf.IsRaw() || mf.IsHEIF() || mf.IsImageOther()) {
+				return nil
+			}
+
+			jobs <- ConvertJob{
+				image:   mf,
+				convert: c,
+			}
 
-		if err != nil {
 			return nil
-		}
-
-		if fileInfo.IsDir() {
-			return nil
-		}
-
-		mf, err := NewMediaFile(fileName)
-
-		if err != nil || !(mf.IsRaw() || mf.IsHEIF() || mf.IsImageOther()) {
-			return nil
-		}
-
-		jobs <- ConvertJob{
-			image:   mf,
-			convert: c,
-		}
-
-		return nil
+		},
+		Unsorted:            true,
+		FollowSymbolicLinks: true,
 	})
 
 	close(jobs)

internal/photoprism/import.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/karrick/godirwalk"
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/event"
@@ -79,81 +80,85 @@ func (imp *Import) Start(opt ImportOptions) {
 
 	indexOpt := IndexOptionsAll()
 
-	err := filepath.Walk(importPath, func(fileName string, fileInfo os.FileInfo, err error) error {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Errorf("import: %s [panic]", err)
-			}
-		}()
+	err := godirwalk.Walk(importPath, &godirwalk.Options{
+		Callback: func(fileName string, info *godirwalk.Dirent) error {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Errorf("import: %s [panic]", err)
+				}
+			}()
 
-		if mutex.Worker.Canceled() {
-			return errors.New("import canceled")
-		}
-
-		if err != nil || done[fileName] {
-			return nil
-		}
-
-		if fileInfo.IsDir() {
-			if fileName != importPath {
-				directories = append(directories, fileName)
+			if mutex.Worker.Canceled() {
+				return errors.New("import canceled")
 			}
 
-			return nil
-		}
-
-		if strings.HasPrefix(filepath.Base(fileName), ".") {
-			done[fileName] = true
-
-			if !opt.RemoveDotFiles {
+			if done[fileName] {
 				return nil
 			}
 
-			if err := os.Remove(fileName); err != nil {
-				log.Errorf("import: could not remove \"%s\" (%s)", fileName, err.Error())
+			if info.IsDir() {
+				if fileName != importPath {
+					directories = append(directories, fileName)
+				}
+			} else if info.IsRegular() && strings.HasPrefix(filepath.Base(fileName), ".") {
+				done[fileName] = true
+
+				if !opt.RemoveDotFiles {
+					return nil
+				}
+
+				if err := os.Remove(fileName); err != nil {
+					log.Errorf("import: could not remove \"%s\" (%s)", fileName, err.Error())
+				}
+
+				return nil
+			}
+
+			if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
+				return result
+			}
+
+			mf, err := NewMediaFile(fileName)
+
+			if err != nil || !mf.IsPhoto() {
+				return nil
+			}
+
+			related, err := mf.RelatedFiles(imp.conf.Settings().Library.GroupRelated)
+
+			if err != nil {
+				event.Error(fmt.Sprintf("import: %s", err.Error()))
+
+				return nil
+			}
+
+			var files MediaFiles
+
+			for _, f := range related.Files {
+				if done[f.FileName()] {
+					continue
+				}
+
+				files = append(files, f)
+				done[f.FileName()] = true
+			}
+
+			done[mf.FileName()] = true
+
+			related.Files = files
+
+			jobs <- ImportJob{
+				FileName:  fileName,
+				Related:   related,
+				IndexOpt:  indexOpt,
+				ImportOpt: opt,
+				Imp:       imp,
 			}
 
 			return nil
-		}
-
-		mf, err := NewMediaFile(fileName)
-
-		if err != nil || !mf.IsPhoto() {
-			return nil
-		}
-
-		related, err := mf.RelatedFiles(imp.conf.Settings().Library.GroupRelated)
-
-		if err != nil {
-			event.Error(fmt.Sprintf("import: %s", err.Error()))
-
-			return nil
-		}
-
-		var files MediaFiles
-
-		for _, f := range related.Files {
-			if done[f.FileName()] {
-				continue
-			}
-
-			files = append(files, f)
-			done[f.FileName()] = true
-		}
-
-		done[mf.FileName()] = true
-
-		related.Files = files
-
-		jobs <- ImportJob{
-			FileName:  fileName,
-			Related:   related,
-			IndexOpt:  indexOpt,
-			ImportOpt: opt,
-			Imp:       imp,
-		}
-
-		return nil
+		},
+		Unsorted:            false,
+		FollowSymbolicLinks: true,
 	})
 
 	close(jobs)

internal/photoprism/index.go

@@ -3,12 +3,10 @@ package photoprism
 import (
 	"errors"
 	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
 	"sync"
 
 	"github.com/jinzhu/gorm"
+	"github.com/karrick/godirwalk"
 	"github.com/photoprism/photoprism/internal/classify"
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/event"
@@ -89,68 +87,62 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
 		}()
 	}
 
-	err := filepath.Walk(originalsPath, func(fileName string, fileInfo os.FileInfo, err error) error {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Errorf("index: %s [panic]", err)
-			}
-		}()
+	err := godirwalk.Walk(originalsPath, &godirwalk.Options{
+		Callback: func(fileName string, info *godirwalk.Dirent) error {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Errorf("index: %s [panic]", err)
+				}
+			}()
 
-		if mutex.Worker.Canceled() {
-			return errors.New("indexing canceled")
-		}
-
-		if err != nil || done[fileName] {
-			return nil
-		}
-
-		hidden := strings.HasPrefix(filepath.Base(fileName), ".")
-
-		if fileInfo.IsDir() && hidden {
-			return filepath.SkipDir
-		}
-
-		if fileInfo.IsDir() || hidden {
-			return nil
-		}
-
-		mf, err := NewMediaFile(fileName)
-
-		if err != nil || !mf.IsPhoto() {
-			return nil
-		}
-
-		related, err := mf.RelatedFiles(ind.conf.Settings().Library.GroupRelated)
-
-		if err != nil {
-			log.Warnf("index: %s", err.Error())
-
-			return nil
-		}
-
-		var files MediaFiles
-
-		for _, f := range related.Files {
-			if done[f.FileName()] {
-				continue
+			if mutex.Worker.Canceled() {
+				return errors.New("indexing canceled")
 			}
 
-			files = append(files, f)
-			done[f.FileName()] = true
-		}
+			if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
+				return result
+			}
 
-		done[mf.FileName()] = true
+			mf, err := NewMediaFile(fileName)
 
-		related.Files = files
+			if err != nil || !mf.IsPhoto() {
+				return nil
+			}
 
-		jobs <- IndexJob{
-			FileName: mf.FileName(),
-			Related:  related,
-			IndexOpt: options,
-			Ind:      ind,
-		}
+			related, err := mf.RelatedFiles(ind.conf.Settings().Library.GroupRelated)
 
-		return nil
+			if err != nil {
+				log.Warnf("index: %s", err.Error())
+
+				return nil
+			}
+
+			var files MediaFiles
+
+			for _, f := range related.Files {
+				if done[f.FileName()] {
+					continue
+				}
+
+				files = append(files, f)
+				done[f.FileName()] = true
+			}
+
+			done[mf.FileName()] = true
+
+			related.Files = files
+
+			jobs <- IndexJob{
+				FileName: mf.FileName(),
+				Related:  related,
+				IndexOpt: options,
+				Ind:      ind,
+			}
+
+			return nil
+		},
+		Unsorted:            false,
+		FollowSymbolicLinks: true,
 	})
 
 	close(jobs)

internal/photoprism/resample.go

@@ -2,14 +2,14 @@ package photoprism
 
 import (
 	"errors"
-	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 
+	"github.com/karrick/godirwalk"
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/mutex"
+	"github.com/photoprism/photoprism/pkg/fs"
 )
 
 // Resample represents a thumbnail generator.
@@ -46,42 +46,48 @@ func (rs *Resample) Start(force bool) error {
 		}()
 	}
 
-	err := filepath.Walk(originalsPath, func(filename string, fileInfo os.FileInfo, err error) error {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Errorf("resample: %s [panic]", err)
+	done := make(map[string]bool)
+
+	err := godirwalk.Walk(originalsPath, &godirwalk.Options{
+		Callback: func(fileName string, info *godirwalk.Dirent) error {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Errorf("resample: %s [panic]", err)
+				}
+			}()
+
+			if mutex.Worker.Canceled() {
+				return errors.New("resample: canceled")
 			}
-		}()
 
-		if mutex.Worker.Canceled() {
-			return errors.New("resample: canceled")
-		}
+			if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
+				return result
+			}
+
+			mf, err := NewMediaFile(fileName)
+
+			if err != nil || !mf.IsJpeg() {
+				return nil
+			}
+
+			relativeName := mf.RelativeName(originalsPath)
+
+			event.Publish("index.thumbnails", event.Data{
+				"fileName": relativeName,
+				"baseName": filepath.Base(relativeName),
+				"force":    force,
+			})
+
+			jobs <- ResampleJob{
+				mediaFile: mf,
+				path:      thumbnailsPath,
+				force:     force,
+			}
 
-		if err != nil || fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
 			return nil
-		}
-
-		mf, err := NewMediaFile(filename)
-
-		if err != nil || !mf.IsJpeg() {
-			return nil
-		}
-
-		fileName := mf.RelativeName(originalsPath)
-
-		event.Publish("index.thumbnails", event.Data{
-			"fileName": fileName,
-			"baseName": filepath.Base(fileName),
-			"force":    force,
-		})
-
-		jobs <- ResampleJob{
-			mediaFile: mf,
-			path:      thumbnailsPath,
-			force:     force,
-		}
-
-		return nil
+		},
+		Unsorted:            true,
+		FollowSymbolicLinks: true,
 	})
 
 	close(jobs)

pkg/fs/walk.go

@@ -0,0 +1,42 @@
+package fs
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/karrick/godirwalk"
+)
+
+// SkipGodirwalk returns true if the file or directory should be skipped in godirwalk.Walk()
+func SkipGodirwalk(fileName string, info *godirwalk.Dirent, done map[string]bool) (skip bool, result error) {
+	isDone := done[fileName]
+	isHidden := strings.HasPrefix(filepath.Base(fileName), ".")
+	isDir := info.IsDir()
+	isSymlink := info.IsSymlink()
+
+	done[fileName] = true
+
+	if isSymlink {
+		skip = true
+
+		// Symlink points to directory?
+		if link, err := os.Stat(fileName); err == nil && link.IsDir() && (isHidden || isDone || done[link.Name()]) {
+			// Don't traverse symlinks if they are hidden or already done...
+			done[link.Name()] = true
+			result = filepath.SkipDir
+		}
+	} else if isDir {
+		skip = true
+
+		if isHidden || isDone {
+			// Don't traverse directories if they are hidden or already done...
+			result = filepath.SkipDir
+		}
+	} else if isHidden || isDone {
+		// Skip files that are hidden or already done...
+		skip = true
+	}
+
+	return skip, result
+}