beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Backend: Replace filepath.Walk with godirwalk.Walk #267

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-05-01 17:31:42 +02:00
parent 9cbad48972
commit 3e6df5fd5d
7 changed files with 235 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -30,7 +30,7 @@ require (
github.com/guregu/null v3.4.0+incompatible // indirect
github.com/jinzhu/gorm v1.9.5
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/karrick/godirwalk v1.15.6 // indirect
github.com/karrick/godirwalk v1.15.6
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/leandro-lugaresi/hub v1.1.0

2
internal/config/db.go
View file

@ -100,7 +100,7 @@ func (c *Config) connectToDatabase(ctx context.Context) error {
go tidb.Start(ctx, c.TidbServerPath(), c.TidbServerPort(), c.TidbServerHost(), c.Debug())
time.Sleep(2 * time.Second)
time.Sleep(5 * time.Second)
}
for i := 1; i <= 12; i++ {

18
internal/photoprism/convert.go
View file

@ -4,12 +4,12 @@ import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"sync"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
@ -49,7 +49,10 @@ func (c *Convert) Start(path string) error {
}()
}
err := filepath.Walk(path, func(fileName string, fileInfo os.FileInfo, err error) error {
done := make(map[string]bool)
err := godirwalk.Walk(path, &godirwalk.Options{
Callback: func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if err := recover(); err != nil {
log.Errorf("convert: %s [panic]", err)
@ -60,12 +63,8 @@ func (c *Convert) Start(path string) error {
return errors.New("convert: canceled")
}
if err != nil {
return nil
}
if fileInfo.IsDir() {
return nil
if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
return result
}
mf, err := NewMediaFile(fileName)
@ -80,6 +79,9 @@ func (c *Convert) Start(path string) error {
}
return nil
},
Unsorted: true,
FollowSymbolicLinks: true,
})
close(jobs)

21
internal/photoprism/import.go
View file

@ -10,6 +10,7 @@ import (
"strings"
"sync"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/event"
@ -79,7 +80,8 @@ func (imp *Import) Start(opt ImportOptions) {
indexOpt := IndexOptionsAll()
err := filepath.Walk(importPath, func(fileName string, fileInfo os.FileInfo, err error) error {
err := godirwalk.Walk(importPath, &godirwalk.Options{
Callback: func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if err := recover(); err != nil {
log.Errorf("import: %s [panic]", err)
@ -90,19 +92,15 @@ func (imp *Import) Start(opt ImportOptions) {
return errors.New("import canceled")
}
if err != nil || done[fileName] {
if done[fileName] {
return nil
}
if fileInfo.IsDir() {
if info.IsDir() {
if fileName != importPath {
directories = append(directories, fileName)
}
return nil
}
if strings.HasPrefix(filepath.Base(fileName), ".") {
} else if info.IsRegular() && strings.HasPrefix(filepath.Base(fileName), ".") {
done[fileName] = true
if !opt.RemoveDotFiles {
@ -116,6 +114,10 @@ func (imp *Import) Start(opt ImportOptions) {
return nil
}
if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
return result
}
mf, err := NewMediaFile(fileName)
if err != nil || !mf.IsPhoto() {
@ -154,6 +156,9 @@ func (imp *Import) Start(opt ImportOptions) {
}
return nil
},
Unsorted: false,
FollowSymbolicLinks: true,
})
close(jobs)

24
internal/photoprism/index.go
View file

@ -3,12 +3,10 @@ package photoprism
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/jinzhu/gorm"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/classify"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
@ -89,7 +87,8 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
}()
}
err := filepath.Walk(originalsPath, func(fileName string, fileInfo os.FileInfo, err error) error {
err := godirwalk.Walk(originalsPath, &godirwalk.Options{
Callback: func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if err := recover(); err != nil {
log.Errorf("index: %s [panic]", err)
@ -100,18 +99,8 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
return errors.New("indexing canceled")
}
if err != nil || done[fileName] {
return nil
}
hidden := strings.HasPrefix(filepath.Base(fileName), ".")
if fileInfo.IsDir() && hidden {
return filepath.SkipDir
}
if fileInfo.IsDir() || hidden {
return nil
if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
return result
}
mf, err := NewMediaFile(fileName)
@ -151,6 +140,9 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
}
return nil
},
Unsorted: false,
FollowSymbolicLinks: true,
})
close(jobs)

24
internal/photoprism/resample.go
View file

@ -2,14 +2,14 @@ package photoprism
import (
"errors"
"os"
"path/filepath"
"strings"
"sync"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/pkg/fs"
)
// Resample represents a thumbnail generator.
@ -46,7 +46,10 @@ func (rs *Resample) Start(force bool) error {
}()
}
err := filepath.Walk(originalsPath, func(filename string, fileInfo os.FileInfo, err error) error {
done := make(map[string]bool)
err := godirwalk.Walk(originalsPath, &godirwalk.Options{
Callback: func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if err := recover(); err != nil {
log.Errorf("resample: %s [panic]", err)
@ -57,21 +60,21 @@ func (rs *Resample) Start(force bool) error {
return errors.New("resample: canceled")
}
if err != nil || fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
return nil
if skip, result := fs.SkipGodirwalk(fileName, info, done); skip {
return result
}
mf, err := NewMediaFile(filename)
mf, err := NewMediaFile(fileName)
if err != nil || !mf.IsJpeg() {
return nil
}
fileName := mf.RelativeName(originalsPath)
relativeName := mf.RelativeName(originalsPath)
event.Publish("index.thumbnails", event.Data{
"fileName": fileName,
"baseName": filepath.Base(fileName),
"fileName": relativeName,
"baseName": filepath.Base(relativeName),
"force": force,
})
@ -82,6 +85,9 @@ func (rs *Resample) Start(force bool) error {
}
return nil
},
Unsorted: true,
FollowSymbolicLinks: true,
})
close(jobs)

42
pkg/fs/walk.go Normal file
View file

@ -0,0 +1,42 @@
package fs
import (
"os"
"path/filepath"
"strings"
"github.com/karrick/godirwalk"
)
// SkipGodirwalk returns true if the file or directory should be skipped in godirwalk.Walk()
func SkipGodirwalk(fileName string, info *godirwalk.Dirent, done map[string]bool) (skip bool, result error) {
isDone := done[fileName]
isHidden := strings.HasPrefix(filepath.Base(fileName), ".")
isDir := info.IsDir()
isSymlink := info.IsSymlink()
done[fileName] = true
if isSymlink {
skip = true
// Symlink points to directory?
if link, err := os.Stat(fileName); err == nil && link.IsDir() && (isHidden || isDone || done[link.Name()]) {
// Don't traverse symlinks if they are hidden or already done...
done[link.Name()] = true
result = filepath.SkipDir
}
} else if isDir {
skip = true
if isHidden || isDone {
// Don't traverse directories if they are hidden or already done...
result = filepath.SkipDir
}
} else if isHidden || isDone {
// Skip files that are hidden or already done...
skip = true
}
return skip, result
}