People: Remove orphan face crop thumbnails #22
This commit is contained in:
parent
e5631d9d28
commit
11f7e76ca3
|
@ -73,6 +73,11 @@ type File struct {
|
||||||
markers *Markers
|
markers *Markers
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TableName returns the entity database table name.
|
||||||
|
func (File) TableName() string {
|
||||||
|
return "files"
|
||||||
|
}
|
||||||
|
|
||||||
type FileInfos struct {
|
type FileInfos struct {
|
||||||
FileWidth int
|
FileWidth int
|
||||||
FileHeight int
|
FileHeight int
|
||||||
|
|
|
@ -42,7 +42,7 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if err := mutex.MainWorker.Start(); err != nil {
|
if err = mutex.MainWorker.Start(); err != nil {
|
||||||
log.Warnf("cleanup: %s (start)", err.Error())
|
log.Warnf("cleanup: %s (start)", err.Error())
|
||||||
return thumbs, orphans, err
|
return thumbs, orphans, err
|
||||||
}
|
}
|
||||||
|
@ -53,15 +53,19 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
|
||||||
log.Infof("cleanup: dry run, nothing will actually be removed")
|
log.Infof("cleanup: dry run, nothing will actually be removed")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find and remove orphan thumbnail files.
|
var fileHashes, thumbHashes query.HashMap
|
||||||
hashes, err := query.FileHashes()
|
|
||||||
|
|
||||||
if err != nil {
|
// Fetch existing media and thumb file hashes.
|
||||||
|
if fileHashes, err = query.FileHashMap(); err != nil {
|
||||||
|
return thumbs, orphans, err
|
||||||
|
} else if thumbHashes, err = query.ThumbHashMap(); err != nil {
|
||||||
return thumbs, orphans, err
|
return thumbs, orphans, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Thumbnails storage path.
|
||||||
thumbPath := w.conf.ThumbPath()
|
thumbPath := w.conf.ThumbPath()
|
||||||
|
|
||||||
|
// Find and remove orphan thumbnail files.
|
||||||
if err := fastwalk.Walk(thumbPath, func(fileName string, info os.FileMode) error {
|
if err := fastwalk.Walk(thumbPath, func(fileName string, info os.FileMode) error {
|
||||||
base := filepath.Base(fileName)
|
base := filepath.Base(fileName)
|
||||||
|
|
||||||
|
@ -79,7 +83,9 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
|
||||||
hash := base[:i]
|
hash := base[:i]
|
||||||
logName := txt.Quote(fs.RelName(fileName, thumbPath))
|
logName := txt.Quote(fs.RelName(fileName, thumbPath))
|
||||||
|
|
||||||
if ok := hashes[hash]; ok {
|
if ok := fileHashes[hash]; ok {
|
||||||
|
// Do nothing.
|
||||||
|
} else if ok := thumbHashes[hash]; ok {
|
||||||
// Do nothing.
|
// Do nothing.
|
||||||
} else if opt.Dry {
|
} else if opt.Dry {
|
||||||
thumbs++
|
thumbs++
|
||||||
|
|
|
@ -170,22 +170,3 @@ func IndexedFiles() (result FileMap, err error) {
|
||||||
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
type HashMap map[string]bool
|
|
||||||
|
|
||||||
// FileHashes returns a map of all known file hashes.
|
|
||||||
func FileHashes() (result HashMap, err error) {
|
|
||||||
result = make(HashMap)
|
|
||||||
|
|
||||||
var hashes []string
|
|
||||||
|
|
||||||
if err := UnscopedDb().Raw("SELECT file_hash FROM files WHERE file_missing = 0 AND deleted_at IS NULL").Pluck("file_hash", &hashes).Error; err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, hash := range hashes {
|
|
||||||
result[hash] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
|
|
86
internal/query/files_hashes.go
Normal file
86
internal/query/files_hashes.go
Normal file
|
@ -0,0 +1,86 @@
|
||||||
|
package query
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
|
||||||
|
"github.com/photoprism/photoprism/internal/entity"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HashMap map[string]bool
|
||||||
|
|
||||||
|
// CountFileHashes counts distinct file hashes.
|
||||||
|
func CountFileHashes() (count int) {
|
||||||
|
if err := UnscopedDb().
|
||||||
|
Table(entity.File{}.TableName()).
|
||||||
|
Where("file_missing = 0 AND deleted_at IS NULL").
|
||||||
|
Select("COUNT(DISTINCT(file_hash))").Count(&count).Error; err != nil {
|
||||||
|
log.Errorf("files: %s (count hashes)", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
// FetchHashMap populates a hash map from the database.
|
||||||
|
func FetchHashMap(rows *sql.Rows, result HashMap, hashLen int) (err error) {
|
||||||
|
defer func(rows *sql.Rows) {
|
||||||
|
err = rows.Close()
|
||||||
|
}(rows)
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var h string
|
||||||
|
|
||||||
|
if err = rows.Scan(&h); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(h) > hashLen {
|
||||||
|
result[h[:hashLen]] = true
|
||||||
|
} else if h != "" {
|
||||||
|
result[h] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileHashMap returns a map of all known file hashes.
|
||||||
|
func FileHashMap() (result HashMap, err error) {
|
||||||
|
count := CountFileHashes()
|
||||||
|
|
||||||
|
result = make(HashMap, count)
|
||||||
|
|
||||||
|
if rows, err := UnscopedDb().
|
||||||
|
Table(entity.File{}.TableName()).
|
||||||
|
Where("file_missing = 0 AND deleted_at IS NULL").
|
||||||
|
Where("file_hash IS NOT NULL AND file_hash <> ''").
|
||||||
|
Select("file_hash").Rows(); err != nil {
|
||||||
|
return result, err
|
||||||
|
} else if err := FetchHashMap(rows, result, 40); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ThumbHashMap returns a map of all known thumb file hashes.
|
||||||
|
func ThumbHashMap() (result HashMap, err error) {
|
||||||
|
tables := []string{
|
||||||
|
entity.Album{}.TableName(),
|
||||||
|
entity.Label{}.TableName(),
|
||||||
|
entity.Marker{}.TableName(),
|
||||||
|
entity.Subject{}.TableName(),
|
||||||
|
}
|
||||||
|
|
||||||
|
result = make(HashMap)
|
||||||
|
|
||||||
|
for i := range tables {
|
||||||
|
if rows, err := UnscopedDb().
|
||||||
|
Table(tables[i]).
|
||||||
|
Where("thumb IS NOT NULL AND thumb <> ''").
|
||||||
|
Select("thumb").Rows(); err != nil {
|
||||||
|
return result, err
|
||||||
|
} else if err := FetchHashMap(rows, result, 40); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
43
internal/query/files_hashes_test.go
Normal file
43
internal/query/files_hashes_test.go
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
package query
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCountFileHashes(t *testing.T) {
|
||||||
|
count := CountFileHashes()
|
||||||
|
|
||||||
|
t.Logf("FILE HASH COUNT: %d", count)
|
||||||
|
|
||||||
|
assert.LessOrEqual(t, 30, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileHashMap(t *testing.T) {
|
||||||
|
result, err := FileHashMap()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("%d FILE HASHES: %#v", len(result), result)
|
||||||
|
|
||||||
|
if len(result) < 3 {
|
||||||
|
t.Fatalf("at least 3 file hashes expected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestThumbHashMap(t *testing.T) {
|
||||||
|
result, err := ThumbHashMap()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("THUMB HASHES: %#v", result)
|
||||||
|
|
||||||
|
if len(result) < 1 {
|
||||||
|
t.Fatalf("at least one thumb hashe expected")
|
||||||
|
}
|
||||||
|
}
|
|
@ -233,20 +233,6 @@ func TestIndexedFiles(t *testing.T) {
|
||||||
t.Logf("INDEXED FILES: %#v", result)
|
t.Logf("INDEXED FILES: %#v", result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFileHashes(t *testing.T) {
|
|
||||||
result, err := FileHashes()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(result) < 3 {
|
|
||||||
t.Fatalf("at least 3 file hashes expected")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("FILE HASHES: %#v", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRenameFile(t *testing.T) {
|
func TestRenameFile(t *testing.T) {
|
||||||
t.Run("empty name", func(t *testing.T) {
|
t.Run("empty name", func(t *testing.T) {
|
||||||
err := RenameFile("xxx", "", "yyy", "yyy")
|
err := RenameFile("xxx", "", "yyy", "yyy")
|
||||||
|
|
Loading…
Reference in a new issue