Index: Skip updates if there are no changes #3227

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/api/index.go

@@ -63,9 +63,10 @@ func StartIndexing(router *gin.RouterGroup) {
 		// Start indexing.
 		ind := get.Index()
 		lastRun := ind.LastRun()
+		indexStart := time.Now()
 		found, updated := ind.Start(indOpt)
 
-		log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
+		log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
 
 		RemoveFromFolderCache(entity.RootOriginals)
 

internal/commands/faces.go

@@ -250,12 +250,13 @@ func facesIndexAction(ctx *cli.Context) error {
 	settings := conf.Settings()
 
 	if w := get.Index(); w != nil {
+		indexStart := time.Now()
 		convert := settings.Index.Convert && conf.SidecarWritable()
 		opt := photoprism.NewIndexOptions(subPath, true, convert, true, true, true)
 
 		found, updated = w.Start(opt)
 
-		log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
+		log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
 	}
 
 	if w := get.Purge(); w != nil {

internal/commands/index.go

@@ -72,12 +72,13 @@ func indexAction(ctx *cli.Context) error {
 	var updated int
 
 	if w := get.Index(); w != nil {
+		indexStart := time.Now()
 		convert := conf.Settings().Index.Convert && conf.SidecarWritable()
 		opt := photoprism.NewIndexOptions(subPath, ctx.Bool("force"), convert, true, false, !ctx.Bool("archived"))
 
 		found, updated = w.Start(opt)
 
-		log.Infof("index: updated %s", english.Plural(updated, "file", "files"))
+		log.Infof("index: updated %s [%s]", english.Plural(updated, "file", "files"), time.Since(indexStart))
 	}
 
 	if w := get.Purge(); w != nil {

internal/config/config.go

@@ -721,8 +721,8 @@ func (c *Config) OriginalsLimit() int {
 	return c.options.OriginalsLimit
 }
 
-// OriginalsLimitBytes returns the maximum size of originals in bytes.
+// OriginalsByteLimit returns the maximum size of originals in bytes.
-func (c *Config) OriginalsLimitBytes() int64 {
+func (c *Config) OriginalsByteLimit() int64 {
 	if result := c.OriginalsLimit(); result <= 0 {
 		return -1
 	} else {

internal/config/config_test.go

@@ -370,12 +370,12 @@ func TestConfig_OriginalsLimit(t *testing.T) {
 	assert.Equal(t, 800, c.OriginalsLimit())
 }
 
-func TestConfig_OriginalsLimitBytes(t *testing.T) {
+func TestConfig_OriginalsByteLimit(t *testing.T) {
 	c := NewConfig(CliTestContext())
 
-	assert.Equal(t, int64(-1), c.OriginalsLimitBytes())
+	assert.Equal(t, int64(-1), c.OriginalsByteLimit())
 	c.options.OriginalsLimit = 800
-	assert.Equal(t, int64(838860800), c.OriginalsLimitBytes())
+	assert.Equal(t, int64(838860800), c.OriginalsByteLimit())
 }
 
 func TestConfig_ResolutionLimit(t *testing.T) {

internal/photoprism/import_worker.go

@@ -156,8 +156,8 @@ func ImportWorker(jobs <-chan ImportJob) {
 			// Ensure that a JPEG and the configured default thumbnail sizes exist.
 			if jpg, err := f.PreviewImage(); err != nil {
 				log.Error(err)
-			} else if exceeds, actual := jpg.ExceedsResolution(o.ResolutionLimit); exceeds {
+			} else if limitErr, _ := jpg.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-				log.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+				log.Errorf("index: %s", limitErr)
 				continue
 			} else if err := jpg.CreateThumbnails(imp.thumbPath(), false); err != nil {
 				log.Errorf("import: failed creating thumbnails for %s (%s)", clean.Log(f.RootRelName()), err.Error())
@@ -181,11 +181,11 @@ func ImportWorker(jobs <-chan ImportJob) {
 				f := related.Main
 
 				// Enforce file size and resolution limits.
-				if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
+				if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
-					log.Warnf("import: %s exceeds file size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
+					log.Warnf("import: %s", limitErr)
 					continue
-				} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
+				} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-					log.Warnf("import: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+					log.Warnf("import: %s", limitErr)
 					continue
 				}
 
@@ -223,10 +223,10 @@ func ImportWorker(jobs <-chan ImportJob) {
 				done[f.FileName()] = true
 
 				// Show warning if sidecar file exceeds size or resolution limit.
-				if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
+				if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
-					log.Warnf("import: sidecar file %s exceeds size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
+					log.Warnf("import: %s", limitErr)
-				} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
+				} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-					log.Warnf("import: sidecar file %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+					log.Warnf("import: %s", limitErr)
 				}
 
 				// Extract metadata to a JSON file with Exiftool.

internal/photoprism/index.go

@@ -221,20 +221,34 @@ func (ind *Index) Start(o IndexOptions) (found fs.Done, updated int) {
 			related, err := mf.RelatedFiles(ind.conf.Settings().StackSequences())
 
 			if err != nil {
-				log.Warnf("index: %s", err.Error())
+				log.Warnf("index: %s", err)
 				return nil
 			}
 
 			var files MediaFiles
 
+			if related.Main == nil {
+				// Nothing to do.
+				found[fileName] = fs.Processed
+				return nil
+			} else if limitErr, _ := related.Main.ExceedsBytes(o.ByteLimit); limitErr != nil {
+				found[fileName] = fs.Processed
+				log.Warnf("index: %s", limitErr)
+				return nil
+			}
+
 			for _, f := range related.Files {
 				if found[f.FileName()].Processed() {
 					continue
 				}
 
-				if f.FileSize() == 0 || ind.files.Indexed(f.RootRelName(), f.Root(), f.ModTime(), o.Rescan) {
+				if fileSize := f.FileSize(); fileSize == 0 || ind.files.Indexed(f.RootRelName(), f.Root(), f.ModTime(), o.Rescan) {
 					found[f.FileName()] = fs.Found
 					continue
+				} else if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
+					found[f.FileName()] = fs.Found
+					log.Infof("index: %s", limitErr)
+					continue
 				}
 
 				files = append(files, f)
@@ -244,7 +258,7 @@ func (ind *Index) Start(o IndexOptions) (found fs.Done, updated int) {
 
 			found[fileName] = fs.Processed
 
-			if len(files) == 0 || related.Main == nil {
+			if len(files) == 0 {
 				// Nothing to do.
 				return nil
 			}

internal/photoprism/index_main.go

@@ -20,12 +20,12 @@ func IndexMain(related *RelatedFiles, ind *Index, o IndexOptions) (result IndexR
 	f := related.Main
 
 	// Enforce file size and resolution limits.
-	if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
+	if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
-		result.Err = fmt.Errorf("index: %s exceeds file size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
+		result.Err = fmt.Errorf("index: %s", limitErr)
 		result.Status = IndexFailed
 		return result
-	} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
+	} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-		result.Err = fmt.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+		result.Err = fmt.Errorf("index: %s", limitErr)
 		result.Status = IndexFailed
 		return result
 	}
@@ -43,11 +43,11 @@ func IndexMain(related *RelatedFiles, ind *Index, o IndexOptions) (result IndexR
 	// Create JPEG sidecar for media files in other formats so that thumbnails can be created.
 	if o.Convert && f.IsMedia() && !f.HasPreviewImage() {
 		if jpg, err := ind.convert.ToImage(f, false); err != nil {
-			result.Err = fmt.Errorf("index: failed converting %s to jpeg (%s)", clean.Log(f.RootRelName()), err.Error())
+			result.Err = fmt.Errorf("index: failed creating preview for %s (%s)", clean.Log(f.RootRelName()), err.Error())
 			result.Status = IndexFailed
 			return result
-		} else if exceeds, actual := jpg.ExceedsResolution(o.ResolutionLimit); exceeds {
+		} else if limitErr, _ := jpg.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-			result.Err = fmt.Errorf("index: %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+			result.Err = fmt.Errorf("index: %s", limitErr)
 			result.Status = IndexFailed
 			return result
 		} else {

internal/photoprism/index_mediafile_test.go

@@ -43,8 +43,8 @@ func TestIndex_MediaFile(t *testing.T) {
 
 		t.Logf("size in megapixel: %d", mediaFile.Megapixels())
 
-		exceeds, actual := mediaFile.ExceedsResolution(cfg.ResolutionLimit())
+		limitErr, _ := mediaFile.ExceedsResolution(cfg.ResolutionLimit())
-		t.Logf("megapixel limit exceeded: %t, %d / %d MP", exceeds, actual, cfg.ResolutionLimit())
+		t.Logf("index: %s", limitErr)
 
 		assert.Contains(t, words, "marienkäfer")
 		assert.Contains(t, words, "burst")

internal/photoprism/index_options.go

@@ -8,7 +8,7 @@ type IndexOptions struct {
 	Stack           bool
 	FacesOnly       bool
 	SkipArchived    bool
-	OriginalsLimit  int
+	ByteLimit       int64
 	ResolutionLimit int
 }
 
@@ -21,7 +21,7 @@ func NewIndexOptions(path string, rescan, convert, stack, facesOnly, skipArchive
 		Stack:           stack,
 		FacesOnly:       facesOnly,
 		SkipArchived:    skipArchived,
-		OriginalsLimit:  Config().OriginalsLimit(),
+		ByteLimit:       Config().OriginalsByteLimit(),
 		ResolutionLimit: Config().ResolutionLimit(),
 	}
 

internal/photoprism/index_related.go

@@ -51,10 +51,10 @@ func IndexRelated(related RelatedFiles, ind *Index, o IndexOptions) (result Inde
 		done[f.FileName()] = true
 
 		// Show warning if sidecar file exceeds size or resolution limit.
-		if exceeds, actual := f.ExceedsFileSize(o.OriginalsLimit); exceeds {
+		if limitErr, _ := f.ExceedsBytes(o.ByteLimit); limitErr != nil {
-			log.Warnf("index: sidecar file %s exceeds size limit (%d / %d MB)", clean.Log(f.RootRelName()), actual, o.OriginalsLimit)
+			log.Warnf("index: %s", limitErr)
-		} else if exceeds, actual = f.ExceedsResolution(o.ResolutionLimit); exceeds {
+		} else if limitErr, _ = f.ExceedsResolution(o.ResolutionLimit); limitErr != nil {
-			log.Warnf("index: sidecar file %s exceeds resolution limit (%d / %d MP)", clean.Log(f.RootRelName()), actual, o.ResolutionLimit)
+			log.Warnf("index: %s", limitErr)
 		}
 
 		// Extract metadata to a JSON file with Exiftool.
@@ -70,7 +70,7 @@ func IndexRelated(related RelatedFiles, ind *Index, o IndexOptions) (result Inde
 		// Create JPEG sidecar for media files in other formats so that thumbnails can be created.
 		if o.Convert && f.IsMedia() && !f.HasPreviewImage() {
 			if jpg, err := ind.convert.ToImage(f, false); err != nil {
-				result.Err = fmt.Errorf("index: failed converting %s to jpeg (%s)", clean.Log(f.RootRelName()), err.Error())
+				result.Err = fmt.Errorf("index: failed creating preview for %s (%s)", clean.Log(f.RootRelName()), err.Error())
 				result.Status = IndexFailed
 				return result
 			} else {

internal/photoprism/mediafile.go

@@ -22,6 +22,7 @@ import (
 	_ "golang.org/x/image/webp"
 
 	"github.com/djherbis/times"
+	"github.com/dustin/go-humanize"
 	"github.com/mandykoh/prism/meta/autometa"
 
 	"github.com/photoprism/photoprism/internal/entity"
@@ -1082,30 +1083,27 @@ func (m *MediaFile) Megapixels() (resolution int) {
 	return resolution
 }
 
-// ExceedsFileSize checks if the file exceeds the configured file size limit in MB.
+// ExceedsBytes checks if the file exceeds the specified size limit in bytes.
-func (m *MediaFile) ExceedsFileSize(limit int) (exceeds bool, actual int) {
+func (m *MediaFile) ExceedsBytes(bytes int64) (err error, actual int64) {
-	const mega = 1048576
+	if bytes <= 0 {
-
+		return nil, 0
-	if limit <= 0 {
+	} else if actual = m.FileSize(); actual <= 0 || actual <= bytes {
-		return false, actual
+		return nil, actual
-	} else if size := m.FileSize(); size <= 0 {
-		return false, actual
 	} else {
-		actual = int(size / mega)
+		return fmt.Errorf("%s exceeds file size limit (%s / %s)", clean.Log(m.RootRelName()), humanize.Bytes(uint64(actual)), humanize.Bytes(uint64(bytes))), actual
-		return size > int64(limit)*mega, actual
 	}
 }
 
 // ExceedsResolution checks if an image in a natively supported format exceeds the configured resolution limit in megapixels.
-func (m *MediaFile) ExceedsResolution(limit int) (exceeds bool, actual int) {
+func (m *MediaFile) ExceedsResolution(limit int) (err error, actual int) {
 	if limit <= 0 {
-		return false, actual
+		return nil, actual
 	} else if !m.IsImage() {
-		return false, actual
+		return nil, actual
-	} else if actual = m.Megapixels(); actual <= 0 {
+	} else if actual = m.Megapixels(); actual <= 0 || actual <= limit {
-		return false, actual
+		return nil, actual
 	} else {
-		return actual > limit, actual
+		return fmt.Errorf("%s exceeds resolution limit (%d / %d MP)", clean.Log(m.RootRelName()), actual, limit), actual
 	}
 }
 

internal/photoprism/mediafile_test.go

@@ -1950,14 +1950,14 @@ func TestMediaFile_Megapixels(t *testing.T) {
 	})
 }
 
-func TestMediaFile_ExceedsFileSize(t *testing.T) {
+func TestMediaFile_ExceedsBytes(t *testing.T) {
 	t.Run("norway-kjetil-moe.webp", func(t *testing.T) {
 		if f, err := NewMediaFile("testdata/norway-kjetil-moe.webp"); err != nil {
 			t.Fatal(err)
 		} else {
-			result, actual := f.ExceedsFileSize(3)
+			err, actual := f.ExceedsBytes(3145728)
-			assert.False(t, result)
+			assert.NoError(t, err)
-			assert.Equal(t, 0, actual)
+			assert.Equal(t, int64(30320), actual)
 			assert.True(t, f.Ok())
 			assert.False(t, f.Empty())
 		}
@@ -1966,9 +1966,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
 		if f, err := NewMediaFile(conf.ExamplesPath() + "/telegram_2020-01-30_09-57-18.jpg"); err != nil {
 			t.Fatal(err)
 		} else {
-			result, actual := f.ExceedsFileSize(-1)
+			err, actual := f.ExceedsBytes(-1)
-			assert.False(t, result)
+			assert.NoError(t, err)
-			assert.Equal(t, 0, actual)
+			assert.Equal(t, int64(0), actual)
 			assert.True(t, f.Ok())
 			assert.False(t, f.Empty())
 		}
@@ -1977,9 +1977,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
 		if f, err := NewMediaFile(conf.ExamplesPath() + "/6720px_white.jpg"); err != nil {
 			t.Fatal(err)
 		} else {
-			result, actual := f.ExceedsFileSize(0)
+			err, actual := f.ExceedsBytes(0)
-			assert.False(t, result)
+			assert.NoError(t, err)
-			assert.Equal(t, 0, actual)
+			assert.Equal(t, int64(0), actual)
 			assert.True(t, f.Ok())
 			assert.False(t, f.Empty())
 		}
@@ -1988,9 +1988,9 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
 		if f, err := NewMediaFile(conf.ExamplesPath() + "/canon_eos_6d.dng"); err != nil {
 			t.Fatal(err)
 		} else {
-			result, actual := f.ExceedsFileSize(10)
+			err, actual := f.ExceedsBytes(10485760)
-			assert.False(t, result)
+			assert.NoError(t, err)
-			assert.Equal(t, 0, actual)
+			assert.Equal(t, int64(411944), actual)
 			assert.True(t, f.Ok())
 			assert.False(t, f.Empty())
 		}
@@ -1999,15 +1999,14 @@ func TestMediaFile_ExceedsFileSize(t *testing.T) {
 		if f, err := NewMediaFile(conf.ExamplesPath() + "/example.bmp"); err != nil {
 			t.Fatal(err)
 		} else {
-			result, actual := f.ExceedsFileSize(10)
+			err, actual := f.ExceedsBytes(10485760)
-			assert.False(t, result)
+			assert.NoError(t, err)
-			assert.Equal(t, 0, actual)
+			assert.Equal(t, int64(20156), actual)
 			assert.True(t, f.Ok())
 			assert.False(t, f.Empty())
 		}
 	})
 }
-
 func TestMediaFile_DecodeConfig(t *testing.T) {
 	t.Run("6720px_white.jpg", func(t *testing.T) {
 		f, err := NewMediaFile(conf.ExamplesPath() + "/6720px_white.jpg")
@@ -2045,7 +2044,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
 			t.Fatal(err)
 		} else {
 			result, actual := f.ExceedsResolution(3)
-			assert.False(t, result)
+			assert.NoError(t, result)
 			assert.Equal(t, 0, actual)
 		}
 	})
@@ -2054,7 +2053,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
 			t.Fatal(err)
 		} else {
 			result, actual := f.ExceedsResolution(3)
-			assert.False(t, result)
+			assert.NoError(t, result)
 			assert.Equal(t, 1, actual)
 		}
 	})
@@ -2065,19 +2064,19 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		exceeds3, actual3 := f.ExceedsResolution(3)
+		err3, actual3 := f.ExceedsResolution(3)
 
-		assert.True(t, exceeds3)
+		assert.Error(t, err3)
 		assert.Equal(t, 30, actual3)
 
-		exceeds30, actual30 := f.ExceedsResolution(30)
+		err30, actual30 := f.ExceedsResolution(30)
 
-		assert.False(t, exceeds30)
+		assert.NoError(t, err30)
 		assert.Equal(t, 30, actual30)
 
-		exceeds33, actual33 := f.ExceedsResolution(33)
+		err33, actual33 := f.ExceedsResolution(33)
 
-		assert.False(t, exceeds33)
+		assert.NoError(t, err33)
 		assert.Equal(t, 30, actual33)
 	})
 	t.Run("canon_eos_6d.dng", func(t *testing.T) {
@@ -2085,7 +2084,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
 			t.Fatal(err)
 		} else {
 			result, actual := f.ExceedsResolution(3)
-			assert.False(t, result)
+			assert.NoError(t, result)
 			assert.Equal(t, 0, actual)
 		}
 	})
@@ -2094,7 +2093,7 @@ func TestMediaFile_ExceedsResolution(t *testing.T) {
 			t.Fatal(err)
 		} else {
 			result, actual := f.ExceedsResolution(3)
-			assert.False(t, result)
+			assert.NoError(t, result)
 			assert.Equal(t, 0, actual)
 		}
 	})