People: Remove orphan face crop thumbnails #22

internal/entity/file.go

@@ -73,6 +73,11 @@ type File struct {
 	markers         *Markers
 }
 
+// TableName returns the entity database table name.
+func (File) TableName() string {
+	return "files"
+}
+
 type FileInfos struct {
 	FileWidth       int
 	FileHeight      int

internal/photoprism/cleanup.go

@@ -42,7 +42,7 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
 		}
 	}()
 
-	if err := mutex.MainWorker.Start(); err != nil {
+	if err = mutex.MainWorker.Start(); err != nil {
 		log.Warnf("cleanup: %s (start)", err.Error())
 		return thumbs, orphans, err
 	}
@@ -53,15 +53,19 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
 		log.Infof("cleanup: dry run, nothing will actually be removed")
 	}
 
-	// Find and remove orphan thumbnail files.
-	hashes, err := query.FileHashes()
+	var fileHashes, thumbHashes query.HashMap
 
-	if err != nil {
+	// Fetch existing media and thumb file hashes.
+	if fileHashes, err = query.FileHashMap(); err != nil {
+		return thumbs, orphans, err
+	} else if thumbHashes, err = query.ThumbHashMap(); err != nil {
 		return thumbs, orphans, err
 	}
 
+	// Thumbnails storage path.
 	thumbPath := w.conf.ThumbPath()
 
+	// Find and remove orphan thumbnail files.
 	if err := fastwalk.Walk(thumbPath, func(fileName string, info os.FileMode) error {
 		base := filepath.Base(fileName)
 
@@ -79,7 +83,9 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
 		hash := base[:i]
 		logName := txt.Quote(fs.RelName(fileName, thumbPath))
 
-		if ok := hashes[hash]; ok {
+		if ok := fileHashes[hash]; ok {
+			// Do nothing.
+		} else if ok := thumbHashes[hash]; ok {
 			// Do nothing.
 		} else if opt.Dry {
 			thumbs++

internal/query/files.go

@@ -170,22 +170,3 @@ func IndexedFiles() (result FileMap, err error) {
 
 	return result, err
 }
-
-type HashMap map[string]bool
-
-// FileHashes returns a map of all known file hashes.
-func FileHashes() (result HashMap, err error) {
-	result = make(HashMap)
-
-	var hashes []string
-
-	if err := UnscopedDb().Raw("SELECT file_hash FROM files WHERE file_missing = 0 AND deleted_at IS NULL").Pluck("file_hash", &hashes).Error; err != nil {
-		return result, err
-	}
-
-	for _, hash := range hashes {
-		result[hash] = true
-	}
-
-	return result, err
-}

internal/query/files_hashes.go

@@ -0,0 +1,86 @@
+package query
+
+import (
+	"database/sql"
+
+	"github.com/photoprism/photoprism/internal/entity"
+)
+
+type HashMap map[string]bool
+
+// CountFileHashes counts distinct file hashes.
+func CountFileHashes() (count int) {
+	if err := UnscopedDb().
+		Table(entity.File{}.TableName()).
+		Where("file_missing = 0 AND deleted_at IS NULL").
+		Select("COUNT(DISTINCT(file_hash))").Count(&count).Error; err != nil {
+		log.Errorf("files: %s (count hashes)", err)
+	}
+
+	return count
+}
+
+// FetchHashMap populates a hash map from the database.
+func FetchHashMap(rows *sql.Rows, result HashMap, hashLen int) (err error) {
+	defer func(rows *sql.Rows) {
+		err = rows.Close()
+	}(rows)
+
+	for rows.Next() {
+		var h string
+
+		if err = rows.Scan(&h); err != nil {
+			return err
+		} else if len(h) > hashLen {
+			result[h[:hashLen]] = true
+		} else if h != "" {
+			result[h] = true
+		}
+	}
+
+	return nil
+}
+
+// FileHashMap returns a map of all known file hashes.
+func FileHashMap() (result HashMap, err error) {
+	count := CountFileHashes()
+
+	result = make(HashMap, count)
+
+	if rows, err := UnscopedDb().
+		Table(entity.File{}.TableName()).
+		Where("file_missing = 0 AND deleted_at IS NULL").
+		Where("file_hash IS NOT NULL AND file_hash <> ''").
+		Select("file_hash").Rows(); err != nil {
+		return result, err
+	} else if err := FetchHashMap(rows, result, 40); err != nil {
+		return result, err
+	}
+
+	return result, err
+}
+
+// ThumbHashMap returns a map of all known thumb file hashes.
+func ThumbHashMap() (result HashMap, err error) {
+	tables := []string{
+		entity.Album{}.TableName(),
+		entity.Label{}.TableName(),
+		entity.Marker{}.TableName(),
+		entity.Subject{}.TableName(),
+	}
+
+	result = make(HashMap)
+
+	for i := range tables {
+		if rows, err := UnscopedDb().
+			Table(tables[i]).
+			Where("thumb IS NOT NULL AND thumb <> ''").
+			Select("thumb").Rows(); err != nil {
+			return result, err
+		} else if err := FetchHashMap(rows, result, 40); err != nil {
+			return result, err
+		}
+	}
+
+	return result, nil
+}

internal/query/files_hashes_test.go

@@ -0,0 +1,43 @@
+package query
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCountFileHashes(t *testing.T) {
+	count := CountFileHashes()
+
+	t.Logf("FILE HASH COUNT: %d", count)
+
+	assert.LessOrEqual(t, 30, count)
+}
+
+func TestFileHashMap(t *testing.T) {
+	result, err := FileHashMap()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("%d FILE HASHES: %#v", len(result), result)
+
+	if len(result) < 3 {
+		t.Fatalf("at least 3 file hashes expected")
+	}
+}
+
+func TestThumbHashMap(t *testing.T) {
+	result, err := ThumbHashMap()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("THUMB HASHES: %#v", result)
+
+	if len(result) < 1 {
+		t.Fatalf("at least one thumb hashe expected")
+	}
+}

internal/query/files_test.go

@@ -233,20 +233,6 @@ func TestIndexedFiles(t *testing.T) {
 	t.Logf("INDEXED FILES: %#v", result)
 }
 
-func TestFileHashes(t *testing.T) {
-	result, err := FileHashes()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(result) < 3 {
-		t.Fatalf("at least 3 file hashes expected")
-	}
-
-	t.Logf("FILE HASHES: %#v", result)
-}
-
 func TestRenameFile(t *testing.T) {
 	t.Run("empty name", func(t *testing.T) {
 		err := RenameFile("xxx", "", "yyy", "yyy")