beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Backend: Refactor file indexing / skipping

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-07-18 20:58:35 +02:00
parent ee8d9ad919
commit 24cfa1aea2
12 changed files with 69 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
internal/entity/photo.go
View file

@ -211,10 +211,10 @@ func (m *Photo) Save() error {
if err := UnscopedDb().Save(m).Error; err == nil {
// Nothing to do.
} else if !strings.Contains(strings.ToLower(err.Error()), "lock") {
log.Errorf("photo: %s (save %s)", err, m.PhotoUID)
log.Debugf("photo: %s (save %s)", err, m.PhotoUID)
return err
} else if err := UnscopedDb().Save(m).Error; err != nil {
log.Errorf("photo: %s (save %s after deadlock)", err, m.PhotoUID)
log.Debugf("photo: %s (save %s after deadlock)", err, m.PhotoUID)
return err
}

4
internal/photoprism/convert.go
View file

@ -53,7 +53,7 @@ func (c *Convert) Start(path string) error {
}()
}
done := make(map[string]bool)
done := make(fs.Done)
ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
if err := ignore.Dir(path); err != nil {
@ -89,7 +89,7 @@ func (c *Convert) Start(path string) error {
return nil
}
done[fileName] = true
done[fileName] = fs.Processed
jobs <- ConvertJob{
image: mf,

10
internal/photoprism/import.go
View file

@ -48,9 +48,9 @@ func (imp *Import) thumbPath() string {
}
// Start imports media files from a directory and converts/indexes them as needed.
func (imp *Import) Start(opt ImportOptions) map[string]bool {
func (imp *Import) Start(opt ImportOptions) fs.Done {
var directories []string
done := make(map[string]bool)
done := make(fs.Done)
ind := imp.index
importPath := opt.Path
@ -145,15 +145,15 @@ func (imp *Import) Start(opt ImportOptions) map[string]bool {
var files MediaFiles
for _, f := range related.Files {
if done[f.FileName()] {
if done[f.FileName()].Processed() {
continue
}
files = append(files, f)
done[f.FileName()] = true
done[f.FileName()] = fs.Processed
}
done[fileName] = true
done[fileName] = fs.Processed
related.Files = files

13
internal/photoprism/index.go
View file

@ -55,14 +55,14 @@ func (ind *Index) Cancel() {
}
// Start indexes media files in the originals directory.
func (ind *Index) Start(opt IndexOptions) map[string]bool {
func (ind *Index) Start(opt IndexOptions) fs.Done {
defer func() {
if r := recover(); r != nil {
log.Errorf("index: %s (panic)\nstack: %s", r, debug.Stack())
}
}()
done := make(map[string]bool)
done := make(fs.Done)
originalsPath := ind.originalsPath()
optionsPath := filepath.Join(originalsPath, opt.Path)
@ -134,7 +134,7 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool {
return result
}
done[fileName] = true
done[fileName] = fs.Found
if !fs.FileExt.Media(fileName) {
return nil
@ -166,12 +166,17 @@ func (ind *Index) Start(opt IndexOptions) map[string]bool {
var files MediaFiles
for _, f := range related.Files {
if done[f.FileName()].Processed() {
continue
}
files = append(files, f)
filesIndexed++
done[f.FileName()] = true
done[f.FileName()] = fs.Processed
}
filesIndexed++
done[fileName] = fs.Processed
related.Files = files

4
internal/photoprism/index_mediafile.go
View file

@ -635,7 +635,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
file.UpdatedIn = int64(time.Since(start))
if err := file.Save(); err != nil {
log.Errorf("index: %s in %s (save file)", err, logName)
log.Errorf("index: failed updating file %s", logName)
result.Status = IndexFailed
result.Err = err
return result
@ -644,7 +644,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
file.CreatedIn = int64(time.Since(start))
if err := file.Create(); err != nil {
log.Errorf("index: %s in %s (create file)", err, logName)
log.Errorf("index: failed adding file %s", logName)
result.Status = IndexFailed
result.Err = err
return result

6
internal/photoprism/purge.go
View file

@ -37,12 +37,12 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
}
}()
var ignore map[string]bool
var ignore fs.Done
if opt.Ignore != nil {
ignore = opt.Ignore
} else {
ignore = make(map[string]bool)
ignore = make(fs.Done)
}
purgedFiles = make(map[string]bool)
@ -76,7 +76,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
fileName := FileName(file.FileRoot, file.FileName)
if ignore[fileName] || purgedFiles[fileName] {
if ignore[fileName].Exists() || purgedFiles[fileName] {
continue
}

4
internal/photoprism/purge_options.go
View file

@ -1,8 +1,10 @@
package photoprism
import "github.com/photoprism/photoprism/pkg/fs"
type PurgeOptions struct {
Path string
Ignore map[string]bool
Ignore fs.Done
Dry bool
Hard bool
}

4
internal/photoprism/resample.go
View file

@ -55,7 +55,7 @@ func (rs *Resample) Start(force bool) (err error) {
}()
}
done := make(map[string]bool)
done := make(fs.Done)
ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
if err := ignore.Dir(originalsPath); err != nil {
@ -91,7 +91,7 @@ func (rs *Resample) Start(force bool) (err error) {
return nil
}
done[fileName] = true
done[fileName] = fs.Processed
relativeName := mf.RelName(originalsPath)

16
internal/query/folders.go
View file

@ -18,18 +18,14 @@ func FoldersByPath(rootName, rootPath, path string, recursive bool) (folders ent
folders = make(entity.Folders, len(dirs))
for i, dir := range dirs {
folder := entity.FindFolder(rootName, filepath.Join(path, dir))
newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil)
if folder == nil {
newFolder := entity.NewFolder(rootName, filepath.Join(path, dir), nil)
if err := newFolder.Create(); err != nil {
log.Errorf("folders: %s (create folder)", err.Error())
} else {
folders[i] = newFolder
}
} else {
if err := newFolder.Create(); err == nil {
folders[i] = newFolder
} else if folder := entity.FindFolder(rootName, filepath.Join(path, dir)); folder != nil {
folders[i] = *folder
} else {
log.Errorf("folders: %s (create folder)", err)
}
}

18
pkg/fs/done.go Normal file
View file

@ -0,0 +1,18 @@
package fs
type Status int8
const (
Found Status = 1
Processed Status = 2
)
type Done map[string]Status
func (s Status) Exists() bool {
return s > 0
}
func (s Status) Processed() bool {
return s >= Processed
}

10
pkg/fs/walk.go
View file

@ -6,8 +6,8 @@ import (
)
// SkipWalk returns true if the file or directory should be skipped in godirwalk.Walk()
func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, ignore *IgnoreList) (skip bool, result error) {
isDone := done[fileName]
func SkipWalk(fileName string, isDir, isSymlink bool, done Done, ignore *IgnoreList) (skip bool, result error) {
isDone := done[fileName].Exists()
isIgnored := ignore.Ignore(fileName)
if isSymlink {
@ -18,11 +18,11 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
if link, err := os.Stat(fileName); err == nil && link.IsDir() {
resolved, err := filepath.EvalSymlinks(fileName)
if err != nil || isIgnored || isDone || done[resolved] {
if err != nil || isIgnored || isDone || done[resolved].Exists() {
result = filepath.SkipDir
} else {
// Don't traverse symlinks if they are hidden or already done...
done[resolved] = true
done[resolved] = Found
}
}
} else if isDir {
@ -38,7 +38,7 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
}
if skip {
done[fileName] = true
done[fileName] = Found
}
return skip, result

24
pkg/fs/walk_test.go
View file

@ -9,10 +9,10 @@ import (
func TestSkipWalk(t *testing.T) {
t.Run("done", func(t *testing.T) {
done := make(map[string]bool)
done := make(Done)
ignore := NewIgnoreList(".ppignore", true, false)
done["foo.jpg"] = true
done["foo.jpg"] = Found
if skip, result := SkipWalk("testdata/directory", true, false, done, ignore); skip {
assert.Nil(t, result)
@ -20,13 +20,13 @@ func TestSkipWalk(t *testing.T) {
t.Fatal("should be skipped")
}
assert.True(t, done["foo.jpg"])
assert.True(t, done["testdata/directory"])
assert.True(t, done["foo.jpg"].Exists())
assert.True(t, done["testdata/directory"].Exists())
assert.Equal(t, 2, len(done))
})
t.Run("symlink", func(t *testing.T) {
done := make(map[string]bool)
done := make(Done)
ignore := NewIgnoreList(".ppignore", true, false)
if skip, result := SkipWalk("testdata/directory/subdirectory/symlink", false, true, done, ignore); skip {
@ -47,15 +47,15 @@ func TestSkipWalk(t *testing.T) {
t.Fatal("should be skipped")
}
assert.True(t, done["testdata/linked"])
assert.True(t, done["testdata/directory/subdirectory/symlink"])
assert.True(t, done["testdata/directory/subdirectory/symlink/self"])
assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"])
assert.True(t, done["testdata/linked"].Exists())
assert.True(t, done["testdata/directory/subdirectory/symlink"].Exists())
assert.True(t, done["testdata/directory/subdirectory/symlink/self"].Exists())
assert.True(t, done["testdata/directory/subdirectory/symlink/self/self"].Exists())
assert.Equal(t, 4, len(done))
})
t.Run("godirwalk", func(t *testing.T) {
done := make(map[string]bool)
done := make(Done)
var skipped []string
var skippedDirs []string
testPath := "testdata"
@ -75,10 +75,10 @@ func TestSkipWalk(t *testing.T) {
return result
}
done[fileName] = true
done[fileName] = Found
if textName := TypeText.Find(fileName, false); textName != "" {
done[textName] = true
done[textName] = Found
}
return nil