beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Index: Skip updates if there are no changes #3227

Browse source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2023-02-23 03:45:58 +01:00
parent 242c8c54b8
commit 668395909d
13 changed files with 94 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
internal/api/index.go
View file

@ -63,9 +63,10 @@ func StartIndexing(router *gin.RouterGroup) {
// Start indexing.
ind := get.Index()
lastRun := ind.LastRun()
indexStart := time.Now()
found, updated := ind.Start(indOpt)
log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
RemoveFromFolderCache(entity.RootOriginals)

3
internal/commands/faces.go
View file

@ -250,12 +250,13 @@ func facesIndexAction(ctx *cli.Context) error {
settings := conf.Settings()
if w := get.Index(); w != nil {
indexStart := time.Now()
convert := settings.Index.Convert && conf.SidecarWritable()
opt := photoprism.NewIndexOptions(subPath, true, convert, true, true, true)
found, updated = w.Start(opt)
log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
}
if w := get.Purge(); w != nil {

3
internal/commands/index.go
View file

@ -72,12 +72,13 @@ func indexAction(ctx *cli.Context) error {
var updated int
if w := get.Index(); w != nil {
indexStart := time.Now()
convert := conf.Settings().Index.Convert && conf.SidecarWritable()
opt := photoprism.NewIndexOptions(subPath, ctx.Bool("force"), convert, true, false, !ctx.Bool("archived"))
found, updated = w.Start(opt)
log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
}
if w := get.Purge(); w != nil {

4
internal/config/config.go
View file

@ -721,8 +721,8 @@ func (c *Config) OriginalsLimit() int {
return c.options.OriginalsLimit
}
// OriginalsLimitBytes returns the maximum size of originals in bytes.
func (c *Config) OriginalsLimitBytes() int64 {
// OriginalsByteLimit returns the maximum size of originals in bytes.
func (c *Config) OriginalsByteLimit() int64 {
if result := c.OriginalsLimit(); result <= 0 {
return -1
} else {

6
internal/config/config_test.go
View file

@ -370,12 +370,12 @@ func TestConfig_OriginalsLimit(t *testing.T) {
assert.Equal(t, 800, c.OriginalsLimit())
}
func TestConfig_OriginalsLimitBytes(t *testing.T) {
func TestConfig_OriginalsByteLimit(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, int64(-1), c.OriginalsLimitBytes())
assert.Equal(t, int64(-1), c.OriginalsByteLimit())
c.options.OriginalsLimit = 800
assert.Equal(t, int64(838860800), c.OriginalsLimitBytes())
assert.Equal(t, int64(838860800), c.OriginalsByteLimit())
}
func TestConfig_ResolutionLimit(t *testing.T) {

20
internal/photoprism/import_worker.go
View file

@ -156,8 +156,8 @@ func ImportWorker(jobs <-chan ImportJob) {
// Ensure that a JPEG and the configured default thumbnail sizes exist.
if jpg, err := f.PreviewImage(); err != nil {
log.Error(err)
} else if exceeds, actual := jpg.ExceedsResolution(o.ResolutionLimit); exceeds {
log.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
} else if limitErr, _ := jpg.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
log.Errorf("index: %s", limitErr)
continue
} else if err := jpg.CreateThumbnails(imp.thumbPath(), false); err != nil {
log.Errorf("import: failed creating thumbnails for %s (%s)", clean.Log(f.RootRelName()), err.Error())
@ -181,11 +181,11 @@ func ImportWorker(jobs <-chan ImportJob) {
f := related.Main
// Enforce file size and resolution limits.
if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
log.Warnf("import: %s exceeds file size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
log.Warnf("import: %s", limitErr)
continue
} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
log.Warnf("import: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
log.Warnf("import: %s", limitErr)
continue
}
@ -223,10 +223,10 @@ func ImportWorker(jobs <-chan ImportJob) {
done[f.FileName()] = true
// Show warning if sidecar file exceeds size or resolution limit.
if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
log.Warnf("import: sidecar file %s exceeds size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
log.Warnf("import: sidecar file %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
log.Warnf("import: %s", limitErr)
} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
log.Warnf("import: %s", limitErr)
}
// Extract metadata to a JSON file with Exiftool.

20
internal/photoprism/index.go
View file

@ -221,20 +221,34 @@ func (ind *Index) Start(o IndexOptions) (found fs.Done, updated int) {
related, err := mf.RelatedFiles(ind.conf.Settings().StackSequences())
if err != nil {
log.Warnf("index: %s", err.Error())
log.Warnf("index: %s", err)
return nil
}
var files MediaFiles
if related.Main == nil {
// Nothing to do.
found[fileName] = fs.Processed
return nil
} else if limitErr, _ := related.Main.ExceedsBytes(o.ByteLimit); limitErr != nil {
found[fileName] = fs.Processed
log.Warnf("index: %s", limitErr)
return nil
}
for _, f := range related.Files {
if found[f.FileName()].Processed() {
continue
}
if f.FileSize() == 0 || ind.files.Indexed(f.RootRelName(), f.Root(), f.ModTime(), o.Rescan) {
if fileSize := f.FileSize(); fileSize == 0 || ind.files.Indexed(f.RootRelName(), f.Root(), f.ModTime(), o.Rescan) {
found[f.FileName()] = fs.Found
continue
} else if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
found[f.FileName()] = fs.Found
log.Infof("index: %s", limitErr)
continue
}
files = append(files, f)
@ -244,7 +258,7 @@ func (ind *Index) Start(o IndexOptions) (found fs.Done, updated int) {
found[fileName] = fs.Processed
if len(files) == 0 || related.Main == nil {
if len(files) == 0 {
// Nothing to do.
return nil
}

14
internal/photoprism/index_main.go
View file

@ -20,12 +20,12 @@ func IndexMain(related *RelatedFiles, ind *Index, o IndexOptions) (result IndexR
f := related.Main
// Enforce file size and resolution limits.
if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
result.Err = fmt.Errorf("index: %s exceeds file size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
result.Err = fmt.Errorf("index: %s", limitErr)
result.Status = IndexFailed
return result
} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
result.Err = fmt.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
result.Err = fmt.Errorf("index: %s", limitErr)
result.Status = IndexFailed
return result
}
@ -43,11 +43,11 @@ func IndexMain(related *RelatedFiles, ind *Index, o IndexOptions) (result IndexR
// Create JPEG sidecar for media files in other formats so that thumbnails can be created.
if o.Convert && f.IsMedia() && !f.HasPreviewImage() {
if jpg, err := ind.convert.ToImage(f, false); err != nil {
result.Err = fmt.Errorf("index: failed converting %s to jpeg (%s)", clean.Log(f.RootRelName()), err.Error())
result.Err = fmt.Errorf("index: failed creating preview for %s (%s)", clean.Log(f.RootRelName()), err.Error())
result.Status = IndexFailed
return result
} else if exceeds, actual := jpg.ExceedsResolution(o.ResolutionLimit); exceeds {
result.Err = fmt.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
} else if limitErr, _ := jpg.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
result.Err = fmt.Errorf("index: %s", limitErr)
result.Status = IndexFailed
return result
} else {

4
internal/photoprism/index_mediafile_test.go
View file

@ -43,8 +43,8 @@ func TestIndex_MediaFile(t *testing.T) {
t.Logf("size in megapixel: %d", mediaFile.Megapixels())
exceeds, actual := mediaFile.ExceedsResolution(cfg.ResolutionLimit())
t.Logf("megapixel limit exceeded: %t, %d / %d MP", exceeds, actual, cfg.ResolutionLimit())
limitErr, _ := mediaFile.ExceedsResolution(cfg.ResolutionLimit())
t.Logf("index: %s", limitErr)
assert.Contains(t, words, "marienkäfer")
assert.Contains(t, words, "burst")

4
internal/photoprism/index_options.go
View file

@ -8,7 +8,7 @@ type IndexOptions struct {
Stack bool
FacesOnly bool
SkipArchived bool
OriginalsLimit int
ByteLimit int64
ResolutionLimit int
}
@ -21,7 +21,7 @@ func NewIndexOptions(path string, rescan, convert, stack, facesOnly, skipArchive
Stack: stack,
FacesOnly: facesOnly,
SkipArchived: skipArchived,
OriginalsLimit: Config().OriginalsLimit(),
ByteLimit: Config().OriginalsByteLimit(),
ResolutionLimit: Config().ResolutionLimit(),
}

10
internal/photoprism/index_related.go
View file

@ -51,10 +51,10 @@ func IndexRelated(related RelatedFiles, ind *Index, o IndexOptions) (result Inde
done[f.FileName()] = true
// Show warning if sidecar file exceeds size or resolution limit.
if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
log.Warnf("index: sidecar file %s exceeds size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
log.Warnf("index: sidecar file %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
log.Warnf("index: %s", limitErr)
} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
log.Warnf("index: %s", limitErr)
}
// Extract metadata to a JSON file with Exiftool.
@ -70,7 +70,7 @@ func IndexRelated(related RelatedFiles, ind *Index, o IndexOptions) (result Inde
// Create JPEG sidecar for media files in other formats so that thumbnails can be created.
if o.Convert && f.IsMedia() && !f.HasPreviewImage() {
if jpg, err := ind.convert.ToImage(f, false); err != nil {
result.Err = fmt.Errorf("index: failed converting %s to jpeg (%s)", clean.Log(f.RootRelName()), err.Error())
result.Err = fmt.Errorf("index: failed creating preview for %s (%s)", clean.Log(f.RootRelName()), err.Error())
result.Status = IndexFailed
return result
} else {

30
internal/photoprism/mediafile.go
View file

@ -22,6 +22,7 @@ import (
_ "golang.org/x/image/webp"
"github.com/djherbis/times"
"github.com/dustin/go-humanize"
"github.com/mandykoh/prism/meta/autometa"
"github.com/photoprism/photoprism/internal/entity"
@ -1082,30 +1083,27 @@ func (m *MediaFile) Megapixels() (resolution int) {
return resolution
}
// ExceedsFileSize checks if the file exceeds the configured file size limit in MB.
func (m *MediaFile) ExceedsFileSize(limit int) (exceeds bool, actual int) {
const mega = 1048576
if limit <= 0 {
return false, actual
} else if size := m.FileSize(); size <= 0 {
return false, actual
// ExceedsBytes checks if the file exceeds the specified size limit in bytes.
func (m *MediaFile) ExceedsBytes(bytes int64) (err error, actual int64) {
if bytes <= 0 {
return nil, 0
} else if actual = m.FileSize(); actual <= 0 || actual <= bytes {
return nil, actual
} else {
actual = int(size / mega)
return size > int64(limit)*mega, actual
return fmt.Errorf("%s exceeds file size limit (%s / %s)", clean.Log(m.RootRelName()), humanize.Bytes(uint64(actual)), humanize.Bytes(uint64(bytes))), actual
}
}
// ExceedsResolution checks if an image in a natively supported format exceeds the configured resolution limit in megapixels.
func (m *MediaFile) ExceedsResolution(limit int) (exceeds bool, actual int) {
func (m *MediaFile) ExceedsResolution(limit int) (err error, actual int) {
if limit <= 0 {
return false, actual
return nil, actual
} else if !m.IsImage() {
return false, actual
} else if actual = m.Megapixels(); actual <= 0 {
return false, actual
return nil, actual
} else if actual = m.Megapixels(); actual <= 0 || actual <= limit {
return nil, actual
} else {
return actual > limit, actual
return fmt.Errorf("%s exceeds resolution limit (%d / %d MP)", clean.Log(m.RootRelName()), actual, limit), actual
}
}

53
internal/photoprism/mediafile_test.go
View file

@ -1950,14 +1950,14 @@ func TestMediaFile_Megapixels(t *testing.T) {
})
}
func TestMediaFile_ExceedsFileSize(t *testing.T) {
func TestMediaFile_ExceedsBytes(t *testing.T) {
t.Run("norway-kjetil-moe.webp", func(t *testing.T) {
if f, err := NewMediaFile("testdata/norway-kjetil-moe.webp"); err != nil {
t.Fatal(err)
} else {
result, actual := f.ExceedsFileSize(3)
assert.False(t, result)
assert.Equal(t, 0, actual)
err, actual := f.ExceedsBytes(3145728)
assert.NoError(t, err)
assert.Equal(t, int64(30320), actual)
assert.True(t, f.Ok())
assert.False(t, f.Empty())
}
@ -1966,9 +1966,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
if f, err := NewMediaFile(conf.ExamplesPath() + "/telegram_2020-01-30_09-57-18.jpg"); err != nil {
t.Fatal(err)
} else {
result, actual := f.ExceedsFileSize(-1)
assert.False(t, result)
assert.Equal(t, 0, actual)
err, actual := f.ExceedsBytes(-1)
assert.NoError(t, err)
assert.Equal(t, int64(0), actual)
assert.True(t, f.Ok())
assert.False(t, f.Empty())
}
@ -1977,9 +1977,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
if f, err := NewMediaFile(conf.ExamplesPath() + "/6720px_white.jpg"); err != nil {
t.Fatal(err)
} else {
result, actual := f.ExceedsFileSize(0)
assert.False(t, result)
assert.Equal(t, 0, actual)
err, actual := f.ExceedsBytes(0)
assert.NoError(t, err)
assert.Equal(t, int64(0), actual)
assert.True(t, f.Ok())
assert.False(t, f.Empty())
}
@ -1988,9 +1988,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
if f, err := NewMediaFile(conf.ExamplesPath() + "/canon_eos_6d.dng"); err != nil {
t.Fatal(err)
} else {
result, actual := f.ExceedsFileSize(10)
assert.False(t, result)
assert.Equal(t, 0, actual)
err, actual := f.ExceedsBytes(10485760)
assert.NoError(t, err)
assert.Equal(t, int64(411944), actual)
assert.True(t, f.Ok())
assert.False(t, f.Empty())
}
@ -1999,15 +1999,14 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
if f, err := NewMediaFile(conf.ExamplesPath() + "/example.bmp"); err != nil {
t.Fatal(err)
} else {
result, actual := f.ExceedsFileSize(10)
assert.False(t, result)
assert.Equal(t, 0, actual)
err, actual := f.ExceedsBytes(10485760)
assert.NoError(t, err)
assert.Equal(t, int64(20156), actual)
assert.True(t, f.Ok())
assert.False(t, f.Empty())
}
})
}
func TestMediaFile_DecodeConfig(t *testing.T) {
t.Run("6720px_white.jpg", func(t *testing.T) {
f, err := NewMediaFile(conf.ExamplesPath() + "/6720px_white.jpg")
@ -2045,7 +2044,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
t.Fatal(err)
} else {
result, actual := f.ExceedsResolution(3)
assert.False(t, result)
assert.NoError(t, result)
assert.Equal(t, 0, actual)
}
})
@ -2054,7 +2053,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
t.Fatal(err)
} else {
result, actual := f.ExceedsResolution(3)
assert.False(t, result)
assert.NoError(t, result)
assert.Equal(t, 1, actual)
}
})
@ -2065,19 +2064,19 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
t.Fatal(err)
}
exceeds3, actual3 := f.ExceedsResolution(3)
err3, actual3 := f.ExceedsResolution(3)
assert.True(t, exceeds3)
assert.Error(t, err3)
assert.Equal(t, 30, actual3)
exceeds30, actual30 := f.ExceedsResolution(30)
err30, actual30 := f.ExceedsResolution(30)
assert.False(t, exceeds30)
assert.NoError(t, err30)
assert.Equal(t, 30, actual30)
exceeds33, actual33 := f.ExceedsResolution(33)
err33, actual33 := f.ExceedsResolution(33)
assert.False(t, exceeds33)
assert.NoError(t, err33)
assert.Equal(t, 30, actual33)
})
t.Run("canon_eos_6d.dng", func(t *testing.T) {
@ -2085,7 +2084,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
t.Fatal(err)
} else {
result, actual := f.ExceedsResolution(3)
assert.False(t, result)
assert.NoError(t, result)
assert.Equal(t, 0, actual)
}
})
@ -2094,7 +2093,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
t.Fatal(err)
} else {
result, actual := f.ExceedsResolution(3)
assert.False(t, result)
assert.NoError(t, result)
assert.Equal(t, 0, actual)
}
})