From d198a056a7f0f04653d9c18421077eb1151254b9 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sun, 29 Aug 2021 13:26:05 +0200 Subject: [PATCH] People: Improve face matching performance and accuracy #22 By default, matching is now limited to unmatched faces and markers. --- internal/entity/account_fixtures.go | 12 +-- internal/entity/album.go | 10 +-- internal/entity/camera_fixtures.go | 4 +- internal/entity/cell_fixtures.go | 40 ++++----- internal/entity/details.go | 2 +- internal/entity/details_fixtures.go | 12 +-- internal/entity/face.go | 27 +++--- internal/entity/face_fixtures.go | 20 ++--- internal/entity/face_test.go | 2 +- internal/entity/file.go | 2 +- internal/entity/folder.go | 2 +- internal/entity/label_fixtures.go | 44 ++++----- internal/entity/link.go | 6 +- internal/entity/link_test.go | 2 +- internal/entity/marker.go | 77 +++++++++++++--- internal/entity/marker_fixtures.go | 17 +++- internal/entity/marker_test.go | 14 +++ internal/entity/photo.go | 12 +-- internal/entity/photo_merge.go | 6 +- internal/entity/photo_optimize.go | 2 +- internal/entity/place_fixtures.go | 32 +++---- internal/entity/place_test.go | 4 +- internal/entity/subject_fixtures.go | 16 ++-- internal/entity/time.go | 10 +-- internal/entity/time_test.go | 8 +- internal/entity/user.go | 2 +- internal/face/face.go | 8 ++ internal/face/net.go | 6 +- internal/photoprism/faces.go | 18 ++-- internal/photoprism/faces_audit.go | 38 ++++++-- internal/photoprism/faces_cluster.go | 8 +- internal/photoprism/faces_match.go | 118 +++++++++++++++++------- internal/photoprism/faces_optimize.go | 2 +- internal/photoprism/faces_stats.go | 4 +- internal/query/faces.go | 74 +++++++++------- internal/query/faces_test.go | 62 ++++++++++--- internal/query/markers.go | 123 +++++++++++++++++--------- internal/query/markers_test.go | 109 ++++++++++++++++++++--- internal/query/subjects.go | 4 +- internal/query/subjects_test.go | 4 +- 40 files changed, 655 insertions(+), 308 deletions(-) diff --git a/internal/entity/account_fixtures.go b/internal/entity/account_fixtures.go index 9c8a32ea8..7b01b042b 100644 --- a/internal/entity/account_fixtures.go +++ b/internal/entity/account_fixtures.go @@ -27,13 +27,13 @@ var AccountFixtures = AccountMap{ SyncPath: "/Photos", SyncStatus: "refresh", SyncInterval: 3600, - SyncDate: sql.NullTime{Time: Timestamp()}, + SyncDate: sql.NullTime{Time: TimeStamp()}, SyncUpload: true, SyncDownload: true, SyncFilenames: true, SyncRaw: true, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, "webdav-dummy2": { @@ -56,13 +56,13 @@ var AccountFixtures = AccountMap{ SyncPath: "/Photos", SyncStatus: "refresh", SyncInterval: 3600, - SyncDate: sql.NullTime{Time: Timestamp()}, + SyncDate: sql.NullTime{Time: TimeStamp()}, SyncUpload: true, SyncDownload: true, SyncFilenames: true, SyncRaw: true, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, } diff --git a/internal/entity/album.go b/internal/entity/album.go index 9f815c740..3eae262fe 100644 --- a/internal/entity/album.go +++ b/internal/entity/album.go @@ -102,7 +102,7 @@ func AddPhotoToAlbums(photo string, albums []string) (err error) { // NewAlbum creates a new album; default name is current month and year func NewAlbum(albumTitle, albumType string) *Album { - now := Timestamp() + now := TimeStamp() if albumType == "" { albumType = AlbumDefault @@ -128,7 +128,7 @@ func NewFolderAlbum(albumTitle, albumPath, albumFilter string) *Album { return nil } - now := Timestamp() + now := TimeStamp() result := &Album{ AlbumOrder: SortOrderAdded, @@ -150,7 +150,7 @@ func NewMomentsAlbum(albumTitle, albumSlug, albumFilter string) *Album { return nil } - now := Timestamp() + now := TimeStamp() result := &Album{ AlbumOrder: SortOrderOldest, @@ -171,7 +171,7 @@ func NewStateAlbum(albumTitle, albumSlug, albumFilter string) *Album { return nil } - now := Timestamp() + now := TimeStamp() result := &Album{ AlbumOrder: SortOrderNewest, @@ -198,7 +198,7 @@ func NewMonthAlbum(albumTitle, albumSlug string, year, month int) *Album { Public: true, } - now := Timestamp() + now := TimeStamp() result := &Album{ AlbumOrder: SortOrderOldest, diff --git a/internal/entity/camera_fixtures.go b/internal/entity/camera_fixtures.go index 0c04a7545..e85687353 100644 --- a/internal/entity/camera_fixtures.go +++ b/internal/entity/camera_fixtures.go @@ -96,8 +96,8 @@ var CameraFixtures = CameraMap{ CameraType: "", CameraDescription: "", CameraNotes: "", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, } diff --git a/internal/entity/cell_fixtures.go b/internal/entity/cell_fixtures.go index dfa60d658..ae8d15654 100644 --- a/internal/entity/cell_fixtures.go +++ b/internal/entity/cell_fixtures.go @@ -29,8 +29,8 @@ var CellFixtures = CellMap{ CellName: "Adosada Platform", CellCategory: "botanical garden", Place: PlaceFixtures.Pointer("mexico"), - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "caravan park": { ID: s2.TokenPrefix + "1ef75a71a36c", @@ -41,13 +41,13 @@ var CellFixtures = CellMap{ PlaceCity: "Mandeni", PlaceState: "KwaZulu-Natal", PlaceCountry: "za", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, CellName: "Lobotes Caravan Park", CellCategory: "camping", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "zinkwazi": { ID: s2.TokenPrefix + "1ef744d1e28c", @@ -55,8 +55,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("zinkwazi"), CellName: "Zinkwazi Beach", CellCategory: "beach", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "hassloch": { ID: s2.TokenPrefix + "1ef744d1e280", @@ -64,8 +64,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("holidaypark"), CellName: "Holiday Park", CellCategory: "park", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "emptyNameLongCity": { ID: s2.TokenPrefix + "1ef744d1e281", @@ -73,8 +73,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("emptyNameLongCity"), CellName: "", CellCategory: "botanical garden", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "emptyNameShortCity": { ID: s2.TokenPrefix + "1ef744d1e282", @@ -82,8 +82,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("emptyNameShortCity"), CellName: "", CellCategory: "botanical garden", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "veryLongLocName": { ID: s2.TokenPrefix + "1ef744d1e283", @@ -91,8 +91,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("veryLongLocName"), CellName: "longlonglonglonglonglonglonglonglonglonglonglonglongName", CellCategory: "cape", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "mediumLongLocName": { ID: s2.TokenPrefix + "1ef744d1e283", @@ -100,8 +100,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("mediumLongLocName"), CellName: "longlonglonglonglonglongName", CellCategory: "botanical garden", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "Neckarbrücke": { ID: s2.TokenPrefix + "1ef744d1e284", @@ -109,8 +109,8 @@ var CellFixtures = CellMap{ Place: PlaceFixtures.Pointer("Germany"), CellName: "Neckarbrücke", CellCategory: "", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, } diff --git a/internal/entity/details.go b/internal/entity/details.go index b8f05c6d8..923d6f413 100644 --- a/internal/entity/details.go +++ b/internal/entity/details.go @@ -63,7 +63,7 @@ func FirstOrCreateDetails(m *Details) *Details { return m } else if err := Db().Where("photo_id = ?", m.PhotoID).First(&result).Error; err == nil { if m.CreatedAt.IsZero() { - m.CreatedAt = Timestamp() + m.CreatedAt = TimeStamp() } return &result diff --git a/internal/entity/details_fixtures.go b/internal/entity/details_fixtures.go index f4b6f67f2..5ba0dad6d 100644 --- a/internal/entity/details_fixtures.go +++ b/internal/entity/details_fixtures.go @@ -29,8 +29,8 @@ var DetailsFixtures = DetailsMap{ Artist: "Hans", Copyright: "copy", License: "MIT", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), KeywordsSrc: "meta", NotesSrc: "manual", SubjectSrc: "meta", @@ -46,8 +46,8 @@ var DetailsFixtures = DetailsMap{ Artist: "Hans", Copyright: "copy", License: "MIT", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), KeywordsSrc: "", NotesSrc: "", SubjectSrc: "meta", @@ -63,8 +63,8 @@ var DetailsFixtures = DetailsMap{ Artist: "Jens Mander", Copyright: "Copyright 2020", License: "n/a", - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), KeywordsSrc: "meta", NotesSrc: "manual", SubjectSrc: "meta", diff --git a/internal/entity/face.go b/internal/entity/face.go index 111429ce0..b5ebf9a82 100644 --- a/internal/entity/face.go +++ b/internal/entity/face.go @@ -35,10 +35,14 @@ type Face struct { var UnknownFace = Face{ ID: UnknownID, FaceSrc: SrcDefault, + MatchedAt: TimePointer(), SubjectUID: UnknownPerson.SubjectUID, EmbeddingJSON: []byte{}, } +// Faceless can be used as argument to match unmatched face markers. +var Faceless = []string{""} + // CreateUnknownFace initializes the database with a placeholder for unknown faces. func CreateUnknownFace() { _ = UnknownFace.Create() @@ -74,7 +78,10 @@ func (m *Face) SetEmbeddings(embeddings Embeddings) (err error) { s := sha1.Sum(m.EmbeddingJSON) m.ID = base32.StdEncoding.EncodeToString(s[:]) - m.UpdatedAt = Timestamp() + m.UpdatedAt = TimeStamp() + + // Reset match timestamp. + m.MatchedAt = nil if m.CreatedAt.IsZero() { m.CreatedAt = m.UpdatedAt @@ -83,10 +90,9 @@ func (m *Face) SetEmbeddings(embeddings Embeddings) (err error) { return nil } -// UpdateMatchTime updates the match timestamp. -func (m *Face) UpdateMatchTime() error { - matched := Timestamp() - m.MatchedAt = &matched +// Matched updates the match timestamp. +func (m *Face) Matched() error { + m.MatchedAt = TimePointer() return UnscopedDb().Model(m).UpdateColumns(Values{"MatchedAt": m.MatchedAt}).Error } @@ -163,6 +169,7 @@ func (m *Face) ReportCollision(embeddings Embeddings) (reported bool, err error) // Should never happen. return false, fmt.Errorf("collision distance must be positive") } else if dist > 0.2 { + m.MatchedAt = nil m.Collisions++ m.CollisionRadius = dist - 0.1 revise = true @@ -171,7 +178,7 @@ func (m *Face) ReportCollision(embeddings Embeddings) (reported bool, err error) m.Collisions++ } - err = m.Updates(Values{"Collisions": m.Collisions, "CollisionRadius": m.CollisionRadius}) + err = m.Updates(Values{"Collisions": m.Collisions, "CollisionRadius": m.CollisionRadius, "MatchedAt": m.MatchedAt}) if err == nil && revise { var revised Markers @@ -210,11 +217,11 @@ func (m *Face) ReviseMatches() (revised Markers, err error) { } // MatchMarkers finds and references matching markers. -func (m *Face) MatchMarkers() error { +func (m *Face) MatchMarkers(faceIds []string) error { var markers Markers err := Db(). - Where("face_id = '' AND marker_invalid = 0 AND marker_type = ?", MarkerFace). + Where("marker_invalid = 0 AND marker_type = ? AND face_id IN (?)", MarkerFace, faceIds). Find(&markers).Error if err != nil { @@ -223,9 +230,9 @@ func (m *Face) MatchMarkers() error { } for _, marker := range markers { - if ok, _ := m.Match(marker.Embeddings()); !ok { + if ok, dist := m.Match(marker.Embeddings()); !ok { // Ignore. - } else if _, err = marker.SetFace(m); err != nil { + } else if _, err = marker.SetFace(m, dist); err != nil { return err } } diff --git a/internal/entity/face_fixtures.go b/internal/entity/face_fixtures.go index 35d9fe6d4..214952fc5 100644 --- a/internal/entity/face_fixtures.go +++ b/internal/entity/face_fixtures.go @@ -27,8 +27,8 @@ var FaceFixtures = FaceMap{ SampleRadius: 0.8, Samples: 5, Collisions: 1, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "unknown": Face{ ID: "IW2P73ISBCUFPIAWSIOZKRDCHHFHC35S", @@ -39,8 +39,8 @@ var FaceFixtures = FaceMap{ Samples: 1, Collisions: 0, MatchedAt: &editTime, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "joe-biden": Face{ ID: "VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG6", @@ -51,8 +51,8 @@ var FaceFixtures = FaceMap{ Samples: 33, Collisions: 0, CollisionRadius: 0, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "jane-doe": Face{ ID: "VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG7", @@ -63,8 +63,8 @@ var FaceFixtures = FaceMap{ Samples: 3, Collisions: 0, CollisionRadius: 0, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "fa-gr": Face{ ID: "TOSCDXCS4VI3PGIUTCNIQCNI6HSFXQVZ", @@ -75,8 +75,8 @@ var FaceFixtures = FaceMap{ Samples: 4, Collisions: 0, CollisionRadius: 0, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, } diff --git a/internal/entity/face_test.go b/internal/entity/face_test.go index 925ff5aab..dbba5acb7 100644 --- a/internal/entity/face_test.go +++ b/internal/entity/face_test.go @@ -176,7 +176,7 @@ func TestFace_UpdateMatchTime(t *testing.T) { m := NewFace("12345", SrcAuto, Embeddings{}) initialMatchTime := m.MatchedAt assert.Equal(t, initialMatchTime, m.MatchedAt) - m.UpdateMatchTime() + m.Matched() assert.NotEqual(t, initialMatchTime, m.MatchedAt) } diff --git a/internal/entity/file.go b/internal/entity/file.go index 546cfcb8c..753923d74 100644 --- a/internal/entity/file.go +++ b/internal/entity/file.go @@ -216,7 +216,7 @@ func (m *File) Delete(permanently bool) error { // Purge removes a file from the index by marking it as missing. func (m *File) Purge() error { - deletedAt := Timestamp() + deletedAt := TimeStamp() m.FileMissing = true m.FilePrimary = false m.DeletedAt = &deletedAt diff --git a/internal/entity/folder.go b/internal/entity/folder.go index 7d81cb031..2f5c5e9c5 100644 --- a/internal/entity/folder.go +++ b/internal/entity/folder.go @@ -55,7 +55,7 @@ func (m *Folder) BeforeCreate(scope *gorm.Scope) error { // NewFolder creates a new file system directory entity. func NewFolder(root, pathName string, modTime time.Time) Folder { - now := Timestamp() + now := TimeStamp() pathName = strings.Trim(pathName, string(os.PathSeparator)) diff --git a/internal/entity/label_fixtures.go b/internal/entity/label_fixtures.go index e4580bb3f..bda16b3a9 100644 --- a/internal/entity/label_fixtures.go +++ b/internal/entity/label_fixtures.go @@ -40,8 +40,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -57,8 +57,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 2, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -74,8 +74,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 3, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -91,8 +91,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 4, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -108,8 +108,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 5, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -125,8 +125,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -142,8 +142,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -159,8 +159,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -176,8 +176,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 4, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -193,8 +193,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, @@ -210,8 +210,8 @@ var LabelFixtures = LabelMap{ LabelNotes: "", PhotoCount: 1, LabelCategories: []*Label{}, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, New: false, }, diff --git a/internal/entity/link.go b/internal/entity/link.go index 03318f4a3..8b234b198 100644 --- a/internal/entity/link.go +++ b/internal/entity/link.go @@ -40,7 +40,7 @@ func (m *Link) BeforeCreate(scope *gorm.Scope) error { // NewLink creates a sharing link. func NewLink(shareUID string, canComment, canEdit bool) Link { - now := Timestamp() + now := TimeStamp() result := Link{ LinkUID: rnd.PPID('s'), @@ -74,7 +74,7 @@ func (m *Link) Expired() bool { return false } - now := Timestamp() + now := TimeStamp() expires := m.ModifiedAt.Add(Seconds(m.LinkExpires)) return now.After(expires) @@ -120,7 +120,7 @@ func (m *Link) Save() error { return fmt.Errorf("link: empty share token") } - m.ModifiedAt = Timestamp() + m.ModifiedAt = TimeStamp() return Db().Save(m).Error } diff --git a/internal/entity/link_test.go b/internal/entity/link_test.go index 1a326e74e..7c37267eb 100644 --- a/internal/entity/link_test.go +++ b/internal/entity/link_test.go @@ -21,7 +21,7 @@ func TestLink_Expired(t *testing.T) { link := NewLink("st9lxuqxpogaaba1", true, false) - link.ModifiedAt = Timestamp().Add(-7 * Day) + link.ModifiedAt = TimeStamp().Add(-7 * Day) link.LinkExpires = 0 assert.False(t, link.Expired()) diff --git a/internal/entity/marker.go b/internal/entity/marker.go index c8416c4ff..f2072381b 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/photoprism/photoprism/pkg/clusters" + "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/pkg/txt" @@ -28,6 +30,7 @@ type Marker struct { SubjectSrc string `gorm:"type:VARBINARY(8);default:'';" json:"SubjectSrc" yaml:"SubjectSrc,omitempty"` Subject *Subject `gorm:"foreignkey:SubjectUID;association_foreignkey:SubjectUID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Subject,omitempty" yaml:"-"` FaceID string `gorm:"type:VARBINARY(42);index;" json:"FaceID" yaml:"FaceID,omitempty"` + FaceDist float64 `gorm:"default:-1" json:"FaceDist" yaml:"FaceDist,omitempty"` Face *Face `gorm:"foreignkey:FaceID;association_foreignkey:ID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"-" yaml:"-"` EmbeddingsJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"EmbeddingsJSON,omitempty"` embeddings Embeddings `gorm:"-"` @@ -36,6 +39,7 @@ type Marker struct { Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"` W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"` H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"` + Size int `gorm:"default:-1" json:"Size" yaml:"Size,omitempty"` Score int `gorm:"type:SMALLINT" json:"Score" yaml:"Score,omitempty"` MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"` MatchedAt *time.Time `sql:"index" json:"MatchedAt" yaml:"MatchedAt,omitempty"` @@ -48,7 +52,7 @@ var UnknownMarker = NewMarker(0, "", SrcDefault, MarkerUnknown, 0, 0, 0, 0) // TableName returns the entity database table name. func (Marker) TableName() string { - return "markers_dev3" + return "markers_dev5" } // NewMarker creates a new entity. @@ -73,8 +77,11 @@ func NewFaceMarker(f face.Face, fileID uint, refUID string) *Marker { m := NewMarker(fileID, refUID, SrcImage, MarkerFace, pos.X, pos.Y, pos.W, pos.H) + m.MatchedAt = nil + m.FaceDist = -1 m.EmbeddingsJSON = f.EmbeddingsJSON() m.LandmarksJSON = f.RelativeLandmarksJSON() + m.Size = f.Size() m.Score = f.Score return m @@ -122,8 +129,25 @@ func (m *Marker) SaveForm(f form.Marker) error { return nil } +// HasFace tests if the marker already has the best matching face. +func (m *Marker) HasFace(f *Face, dist float64) bool { + if m.FaceID == "" { + return false + } else if f == nil { + return m.FaceID != "" + } else if m.FaceID == f.ID { + return m.FaceID != "" + } else if m.FaceDist < 0 { + return false + } else if dist < 0 { + return true + } + + return m.FaceDist <= dist +} + // SetFace sets a new face for this marker. -func (m *Marker) SetFace(f *Face) (updated bool, err error) { +func (m *Marker) SetFace(f *Face, dist float64) (updated bool, err error) { if f == nil { return false, fmt.Errorf("face is nil") } @@ -154,7 +178,7 @@ func (m *Marker) SetFace(f *Face) (updated bool, err error) { // Skip update if the same face is already set. if m.SubjectUID == f.SubjectUID && m.FaceID == f.ID { // Update matching timestamp. - m.MatchedAt = TimestampPointer() + m.MatchedAt = TimePointer() return false, m.Updates(Values{"MatchedAt": m.MatchedAt}) } @@ -164,10 +188,25 @@ func (m *Marker) SetFace(f *Face) (updated bool, err error) { SubjectSrc := m.SubjectSrc m.FaceID = f.ID + m.FaceDist = dist + + if m.FaceDist < 0 { + faceEmbedding := f.Embedding() + + // Calculate smallest distance to embeddings. + for _, e := range m.Embeddings() { + if len(e) != len(faceEmbedding) { + continue + } + + if d := clusters.EuclideanDistance(e, faceEmbedding); d < m.FaceDist || m.FaceDist < 0 { + m.FaceDist = d + } + } + } if f.SubjectUID != "" { m.SubjectUID = f.SubjectUID - m.SubjectSrc = SrcAuto } if err := m.SyncSubject(false); err != nil { @@ -184,9 +223,9 @@ func (m *Marker) SetFace(f *Face) (updated bool, err error) { updated = m.FaceID != faceID || m.SubjectUID != subjectUID || m.SubjectSrc != SubjectSrc // Update matching timestamp. - m.MatchedAt = TimestampPointer() + m.MatchedAt = TimePointer() - return updated, m.Updates(Values{"FaceID": m.FaceID, "SubjectUID": m.SubjectUID, "SubjectSrc": m.SubjectSrc, "MatchedAt": m.MatchedAt}) + return updated, m.Updates(Values{"FaceID": m.FaceID, "FaceDist": m.FaceDist, "SubjectUID": m.SubjectUID, "SubjectSrc": m.SubjectSrc, "MatchedAt": m.MatchedAt}) } // SyncSubject maintains the marker subject relationship. @@ -303,7 +342,7 @@ func (m *Marker) ClearSubject(src string) error { // Do nothing } else if reported, err := m.Face.ReportCollision(m.Embeddings()); err != nil { return err - } else if err := m.Updates(Values{"MarkerName": "", "FaceID": "", "SubjectUID": "", "SubjectSrc": src}); err != nil { + } else if err := m.Updates(Values{"MarkerName": "", "FaceID": "", "FaceDist": -1.0, "SubjectUID": "", "SubjectSrc": src}); err != nil { return err } else if reported { log.Debugf("faces: collision with %s", m.Face.ID) @@ -313,6 +352,7 @@ func (m *Marker) ClearSubject(src string) error { m.MarkerName = "" m.FaceID = "" + m.FaceDist = -1.0 m.SubjectUID = "" m.SubjectSrc = src @@ -325,12 +365,16 @@ func (m *Marker) GetFace() (f *Face) { return m.Face } + // Add face if size if m.FaceID == "" && m.SubjectSrc == SrcManual { - if f = NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil { + if m.Size < face.ClusterMinSize || m.Score < face.ClusterMinScore { + log.Debugf("faces: skipped adding face for low-quality marker %d, size %d, score %d", m.ID, m.Size, m.Score) + return nil + } else if f = NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil { return nil } else if err := f.Create(); err != nil { return nil - } else if err := f.MatchMarkers(); err != nil { + } else if err := f.MatchMarkers(Faceless); err != nil { log.Errorf("faces: %s (match markers)", err) } @@ -345,10 +389,8 @@ func (m *Marker) GetFace() (f *Face) { // ClearFace removes an existing face association. func (m *Marker) ClearFace() (updated bool, err error) { - m.MatchedAt = TimestampPointer() - if m.FaceID == "" { - return false, m.Updates(Values{"MatchedAt": m.MatchedAt}) + return false, m.Matched() } updated = true @@ -356,18 +398,25 @@ func (m *Marker) ClearFace() (updated bool, err error) { // Remove face references. m.Face = nil m.FaceID = "" + m.MatchedAt = TimePointer() // Remove subject if set automatically. if m.SubjectSrc == SrcAuto { m.SubjectUID = "" - err = m.Updates(Values{"FaceID": "", "SubjectUID": "", "MatchedAt": m.MatchedAt}) + err = m.Updates(Values{"FaceID": "", "FaceDist": -1.0, "SubjectUID": "", "MatchedAt": m.MatchedAt}) } else { - err = m.Updates(Values{"FaceID": "", "MatchedAt": m.MatchedAt}) + err = m.Updates(Values{"FaceID": "", "FaceDist": -1.0, "MatchedAt": m.MatchedAt}) } return updated, err } +// Matched updates the match timestamp. +func (m *Marker) Matched() error { + m.MatchedAt = TimePointer() + return UnscopedDb().Model(m).UpdateColumns(Values{"MatchedAt": m.MatchedAt}).Error +} + // FindMarker returns an existing row if exists. func FindMarker(id uint) *Marker { result := Marker{} diff --git a/internal/entity/marker_fixtures.go b/internal/entity/marker_fixtures.go index 59b60808e..628c1ce01 100644 --- a/internal/entity/marker_fixtures.go +++ b/internal/entity/marker_fixtures.go @@ -29,6 +29,7 @@ var MarkerFixtures = MarkerMap{ Y: 0.206944, W: 0.355556, H: .355556, + Size: 200, Score: 100, }, "1000003-2": Marker{ @@ -43,6 +44,7 @@ var MarkerFixtures = MarkerMap{ Y: 0.106944, W: 0.05, H: 0.05, + Size: 200, Score: 100, }, "1000003-3": Marker{ @@ -56,6 +58,7 @@ var MarkerFixtures = MarkerMap{ Y: 0.5, W: 0, H: 0, + Size: 200, Score: 100, }, "1000003-4": Marker{ @@ -71,6 +74,7 @@ var MarkerFixtures = MarkerMap{ Y: 0.7, W: 0.2, H: 0.05, + Size: 160, Score: 50, }, "1000003-5": Marker{ @@ -88,12 +92,14 @@ var MarkerFixtures = MarkerMap{ Y: 0.3, W: 0.1, H: 0.1, + Size: 200, Score: 50, }, "1000003-6": Marker{ ID: 6, FileID: 1000003, FaceID: FaceFixtures.Get("john-doe").ID, + FaceDist: 0.2, SubjectSrc: SrcAuto, SubjectUID: "", MarkerSrc: SrcImage, @@ -105,12 +111,14 @@ var MarkerFixtures = MarkerMap{ Y: 0.282292, W: 0.285937, H: 0.38125, + Size: 200, Score: 100, }, - "ma-ba-1": Marker{ //file id 31 matcht mit fa-gr1-3 + "ma-ba-1": Marker{ // file id 31 matches with fa-gr1-3 ID: 7, FileID: 1000004, FaceID: FaceFixtures.Get("fa-gr").ID, + FaceDist: 0.5, SubjectSrc: "", SubjectUID: "", MarkerSrc: SrcImage, @@ -122,12 +130,14 @@ var MarkerFixtures = MarkerMap{ Y: 0.681125, W: 0.113281, H: 0.169988, + Size: 240, Score: 243, }, "fa-gr-1": Marker{ // file id 6 ID: 8, FileID: 1000005, FaceID: FaceFixtures.Get("fa-gr").ID, + FaceDist: 0.6, SubjectSrc: SrcAuto, SubjectUID: "", MarkerSrc: SrcImage, @@ -139,12 +149,14 @@ var MarkerFixtures = MarkerMap{ Y: 0.271981, W: 0.234375, H: 0.3517, + Size: 200, Score: 107, }, "fa-gr-2": Marker{ // file id 7 ID: 9, FileID: 1000006, FaceID: FaceFixtures.Get("fa-gr").ID, + FaceDist: 0.6, SubjectSrc: SrcAuto, SubjectUID: "", MarkerSrc: SrcImage, @@ -156,12 +168,14 @@ var MarkerFixtures = MarkerMap{ Y: 0.249707, W: 0.214062, H: 0.321219, + Size: 200, Score: 74, }, "fa-gr-3": Marker{ // file id 8 ID: 10, FileID: 1000007, FaceID: FaceFixtures.Get("fa-gr").ID, + FaceDist: 0.6, SubjectSrc: SrcAuto, SubjectUID: "", MarkerSrc: SrcImage, @@ -173,6 +187,7 @@ var MarkerFixtures = MarkerMap{ Y: 0.240328, W: 0.3625, H: 0.543962, + Size: 200, Score: 56, }, } diff --git a/internal/entity/marker_test.go b/internal/entity/marker_test.go index 9f90a4626..2363abbd2 100644 --- a/internal/entity/marker_test.go +++ b/internal/entity/marker_test.go @@ -253,3 +253,17 @@ func TestMarker_Embeddings(t *testing.T) { assert.Empty(t, m.Embeddings()[0]) }) } + +func TestMarker_HasFace(t *testing.T) { + t.Run("true", func(t *testing.T) { + m := MarkerFixtures.Get("1000003-6") + + assert.True(t, m.HasFace(nil, -1)) + assert.True(t, m.HasFace(FaceFixtures.Pointer("joe-biden"), -1)) + }) + t.Run("false", func(t *testing.T) { + m := MarkerFixtures.Get("1000003-6") + + assert.False(t, m.HasFace(FaceFixtures.Pointer("joe-biden"), 0.1)) + }) +} diff --git a/internal/entity/photo.go b/internal/entity/photo.go index 03c99ff3f..65c73200e 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -174,7 +174,7 @@ func SavePhotoForm(model Photo, form form.Photo) error { log.Errorf("photo: %s", err.Error()) } - edited := Timestamp() + edited := TimeStamp() model.EditedAt = &edited model.PhotoQuality = model.QualityScore() @@ -335,7 +335,7 @@ func (m *Photo) ClassifyLabels() classify.Labels { // BeforeCreate creates a random UID if needed before inserting a new row to the database. func (m *Photo) BeforeCreate(scope *gorm.Scope) error { if m.TakenAt.IsZero() || m.TakenAtLocal.IsZero() { - now := Timestamp() + now := TimeStamp() if err := scope.SetColumn("TakenAt", now); err != nil { return err @@ -356,7 +356,7 @@ func (m *Photo) BeforeCreate(scope *gorm.Scope) error { // BeforeSave ensures the existence of TakenAt properties before indexing or updating a photo func (m *Photo) BeforeSave(scope *gorm.Scope) error { if m.TakenAt.IsZero() || m.TakenAtLocal.IsZero() { - now := Timestamp() + now := TimeStamp() if err := scope.SetColumn("TakenAt", now); err != nil { return err @@ -1041,7 +1041,7 @@ func (m *Photo) AllFiles() (files Files) { // Archive removes the photo from albums and flags it as archived (soft delete). func (m *Photo) Archive() error { - deletedAt := Timestamp() + deletedAt := TimeStamp() if err := Db().Model(&PhotoAlbum{}).Where("photo_uid = ?", m.PhotoUID).UpdateColumn("hidden", true).Error; err != nil { return err @@ -1073,7 +1073,7 @@ func (m *Photo) Delete(permanently bool) error { Db().Delete(File{}, "photo_id = ?", m.ID) - return m.Updates(map[string]interface{}{"DeletedAt": Timestamp(), "PhotoQuality": -1}) + return m.Updates(map[string]interface{}{"DeletedAt": TimeStamp(), "PhotoQuality": -1}) } // Delete permanently deletes the entity from the database. @@ -1135,7 +1135,7 @@ func (m *Photo) Approve() error { return nil } - edited := Timestamp() + edited := TimeStamp() m.EditedAt = &edited m.PhotoQuality = m.QualityScore() diff --git a/internal/entity/photo_merge.go b/internal/entity/photo_merge.go index be9d74568..d72615761 100644 --- a/internal/entity/photo_merge.go +++ b/internal/entity/photo_merge.go @@ -87,10 +87,10 @@ func (m *Photo) Merge(mergeMeta, mergeUuid bool) (original Photo, merged Photos, continue } - deleted := Timestamp() + deleted := TimeStamp() logResult(UnscopedDb().Exec("UPDATE `files` SET photo_id = ?, photo_uid = ?, file_primary = 0 WHERE photo_id = ?", original.ID, original.PhotoUID, merge.ID)) - logResult(UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", Timestamp(), merge.ID)) + logResult(UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", TimeStamp(), merge.ID)) switch DbDialect() { case MySQL: @@ -112,7 +112,7 @@ func (m *Photo) Merge(mergeMeta, mergeUuid bool) (original Photo, merged Photos, } if original.ID != m.ID { - deleted := Timestamp() + deleted := TimeStamp() m.DeletedAt = &deleted m.PhotoQuality = -1 } diff --git a/internal/entity/photo_optimize.go b/internal/entity/photo_optimize.go index 35a3c04fe..8413a56fb 100644 --- a/internal/entity/photo_optimize.go +++ b/internal/entity/photo_optimize.go @@ -51,7 +51,7 @@ func (m *Photo) Optimize(mergeMeta, mergeUuid, estimatePlace bool) (updated bool m.PhotoQuality = m.QualityScore() - checked := Timestamp() + checked := TimeStamp() if reflect.DeepEqual(*m, current) { return false, merged, m.Update("CheckedAt", &checked) diff --git a/internal/entity/place_fixtures.go b/internal/entity/place_fixtures.go index 18a966af7..b1af3dfbc 100644 --- a/internal/entity/place_fixtures.go +++ b/internal/entity/place_fixtures.go @@ -32,8 +32,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "ancient, pyramid", PlaceFavorite: false, PhotoCount: 1, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "zinkwazi": { ID: s2.TokenPrefix + "1ef744d1e279", @@ -44,8 +44,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "holidaypark": { ID: s2.TokenPrefix + "1ef744d1e280", @@ -56,8 +56,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "emptyNameLongCity": { ID: s2.TokenPrefix + "1ef744d1e281", @@ -68,8 +68,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "emptyNameShortCity": { ID: s2.TokenPrefix + "1ef744d1e282", @@ -80,8 +80,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "veryLongLocName": { ID: s2.TokenPrefix + "1ef744d1e283", @@ -92,8 +92,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "mediumLongLocName": { ID: s2.TokenPrefix + "1ef744d1e284", @@ -104,8 +104,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: true, PhotoCount: 2, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, "Germany": { ID: s2.TokenPrefix + "1ef744d1e285", @@ -116,8 +116,8 @@ var PlaceFixtures = PlacesMap{ PlaceKeywords: "", PlaceFavorite: false, PhotoCount: 1, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), }, } diff --git a/internal/entity/place_test.go b/internal/entity/place_test.go index 626eb370e..e24412355 100644 --- a/internal/entity/place_test.go +++ b/internal/entity/place_test.go @@ -72,8 +72,8 @@ func TestPlace_Find(t *testing.T) { PlaceKeywords: "", PlaceFavorite: false, PhotoCount: 0, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), } err := place.Find() assert.EqualError(t, err, "record not found") diff --git a/internal/entity/subject_fixtures.go b/internal/entity/subject_fixtures.go index a67de3388..4160d39a8 100644 --- a/internal/entity/subject_fixtures.go +++ b/internal/entity/subject_fixtures.go @@ -32,8 +32,8 @@ var SubjectFixtures = SubjectMap{ SubjectNotes: "Short Note", MetadataJSON: []byte(""), PhotoCount: 1, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, "joe-biden": Subject{ @@ -49,8 +49,8 @@ var SubjectFixtures = SubjectMap{ SubjectNotes: "", MetadataJSON: []byte(""), PhotoCount: 1, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, "dangling": Subject{ @@ -66,8 +66,8 @@ var SubjectFixtures = SubjectMap{ SubjectNotes: "", MetadataJSON: []byte(""), PhotoCount: 0, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, "jane-doe": Subject{ @@ -83,8 +83,8 @@ var SubjectFixtures = SubjectMap{ SubjectNotes: "", MetadataJSON: []byte(""), PhotoCount: 3, - CreatedAt: Timestamp(), - UpdatedAt: Timestamp(), + CreatedAt: TimeStamp(), + UpdatedAt: TimeStamp(), DeletedAt: nil, }, } diff --git a/internal/entity/time.go b/internal/entity/time.go index 3515c4c94..c9f2009d0 100644 --- a/internal/entity/time.go +++ b/internal/entity/time.go @@ -4,14 +4,14 @@ import "time" const Day = time.Hour * 24 -// Timestamp returns the current time in UTC rounded to seconds. -func Timestamp() time.Time { +// TimeStamp returns the current timestamp in UTC rounded to seconds. +func TimeStamp() time.Time { return time.Now().UTC().Round(time.Second) } -// TimestampPointer returns a current timestamp pointer. -func TimestampPointer() *time.Time { - t := Timestamp() +// TimePointer returns a pointer to the current timestamp. +func TimePointer() *time.Time { + t := TimeStamp() return &t } diff --git a/internal/entity/time_test.go b/internal/entity/time_test.go index f4e4e0361..999e985a0 100644 --- a/internal/entity/time_test.go +++ b/internal/entity/time_test.go @@ -5,8 +5,8 @@ import ( "time" ) -func TestTimestamp(t *testing.T) { - result := Timestamp() +func TestTimeStamp(t *testing.T) { + result := TimeStamp() if result.Location() != time.UTC { t.Fatal("timestamp zone must be utc") @@ -17,8 +17,8 @@ func TestTimestamp(t *testing.T) { } } -func TestTimestampPointer(t *testing.T) { - result := TimestampPointer() +func TestTimePointer(t *testing.T) { + result := TimePointer() if result == nil { t.Fatal("result must not be nil") diff --git a/internal/entity/user.go b/internal/entity/user.go index ed9b32825..f51bdab48 100644 --- a/internal/entity/user.go +++ b/internal/entity/user.go @@ -308,7 +308,7 @@ func (m *User) InvalidPassword(password string) bool { return true } - if err := Db().Model(m).Updates(map[string]interface{}{"login_attempts": 0, "login_at": Timestamp()}).Error; err != nil { + if err := Db().Model(m).Updates(map[string]interface{}{"login_attempts": 0, "login_at": TimeStamp()}).Error; err != nil { log.Errorf("user: %s (update last login)", err) } diff --git a/internal/face/face.go b/internal/face/face.go index f8ed925ac..dc565bd1d 100644 --- a/internal/face/face.go +++ b/internal/face/face.go @@ -38,8 +38,11 @@ import ( "github.com/photoprism/photoprism/internal/event" ) +var CropSize = 160 var ClusterCore = 4 var ClusterRadius = 0.6 +var ClusterMinScore = 30 +var ClusterMinSize = CropSize var SampleThreshold = 2 * ClusterCore var log = event.Log @@ -103,6 +106,11 @@ type Face struct { Embeddings [][]float32 `json:"embeddings,omitempty"` } +// Size returns the absolute face size in pixels. +func (f *Face) Size() int { + return f.Face.Scale +} + // Dim returns the max number of rows and cols as float32 to calculate relative coordinates. func (f *Face) Dim() float32 { if f.Cols > 0 { diff --git a/internal/face/net.go b/internal/face/net.go index fc29c2e6d..9d9818248 100644 --- a/internal/face/net.go +++ b/internal/face/net.go @@ -130,7 +130,7 @@ func (t *Net) getFaceCrop(fileName, fileHash string, f Point) (img image.Image, } img = imaging.Crop(img, image.Rect(y, x, y+f.Scale, x+f.Scale)) - img = imaging.Fill(img, 160, 160, imaging.Center, imaging.Lanczos) + img = imaging.Fill(img, CropSize, CropSize, imaging.Center, imaging.Lanczos) if err := imaging.Save(img, cacheFile); err != nil { log.Errorf("faces: failed caching crop %s", filepath.Base(cacheFile)) @@ -142,13 +142,13 @@ func (t *Net) getFaceCrop(fileName, fileHash string, f Point) (img image.Image, } func (t *Net) getEmbeddings(img image.Image) [][]float32 { - tensor, err := imageToTensor(img, 160, 160) + tensor, err := imageToTensor(img, CropSize, CropSize) if err != nil { log.Errorf("faces: failed to convert image to tensor: %v", err) } - // TODO: prewhiten image as in facenet + // TODO: pre-whiten image as in facenet trainPhaseBoolTensor, err := tf.NewTensor(false) diff --git a/internal/photoprism/faces.go b/internal/photoprism/faces.go index 5e662a2c6..9ba1c37e3 100644 --- a/internal/photoprism/faces.go +++ b/internal/photoprism/faces.go @@ -51,22 +51,22 @@ func (w *Faces) Start(opt FacesOptions) (err error) { defer mutex.FacesWorker.Stop() - // Remove invalid reference IDs from markers table. - if removed, err := query.RemoveInvalidMarkerReferences(); err != nil { - log.Errorf("faces: %s (remove invalid references)", err) + // Repair invalid marker face and subject references. + if removed, err := query.FixMarkerReferences(); err != nil { + log.Errorf("faces: %s (fix references)", err) } else if removed > 0 { - log.Infof("faces: removed %d invalid references", removed) + log.Infof("faces: fixed %d marker references", removed) } else { - log.Debugf("faces: no invalid references") + log.Debugf("faces: no invalid marker references") } - // Add known marker subjects. - if affected, err := query.AddFaceMarkerSubjects(); err != nil { - log.Errorf("faces: %s (match markers with subjects)", err) + // Create known marker subjects if needed. + if affected, err := query.CreateMarkerSubjects(); err != nil { + log.Errorf("faces: %s (create subjects)", err) } else if affected > 0 { log.Infof("faces: added %d known marker subjects", affected) } else { - log.Debugf("faces: no subjects were missing") + log.Debugf("faces: marker subjects already exist") } // Optimize existing face clusters. diff --git a/internal/photoprism/faces_audit.go b/internal/photoprism/faces_audit.go index 982906f7f..37fbf42a2 100644 --- a/internal/photoprism/faces_audit.go +++ b/internal/photoprism/faces_audit.go @@ -9,7 +9,7 @@ import ( // Audit face clusters and subjects. func (w *Faces) Audit(fix bool) (err error) { - invalidFaces, invalidSubj, err := query.MarkersWithInvalidReferences() + invalidFaces, invalidSubj, err := query.MarkersWithNonExistentReferences() if err != nil { return err @@ -21,15 +21,37 @@ func (w *Faces) Audit(fix bool) (err error) { log.Error(err) } - log.Infof("%d subjects indexed", len(subj)) + if n := len(subj); n == 0 { + log.Infof("found no subjects") + } else { + log.Infof("%d known subjects", n) + } - log.Infof("%d markers with non-existent subjects", len(invalidSubj)) + // Fix non-existent marker subjects references? + if n := len(invalidSubj); n == 0 { + log.Infof("found no invalid marker subjects") + } else if !fix { + log.Infof("%d markers with non-existent subjects", n) + } else if removed, err := query.RemoveNonExistentMarkerSubjects(); err != nil { + log.Infof("removed %d / %d markers with non-existent subjects", removed, n) + } else { + log.Error(err) + } - log.Infof("%d markers with non-existent faces", len(invalidFaces)) + // Fix non-existent marker face references? + if n := len(invalidFaces); n == 0 { + log.Infof("found no invalid marker faces") + } else if !fix { + log.Infof("%d markers with non-existent faces", n) + } else if removed, err := query.RemoveNonExistentMarkerFaces(); err != nil { + log.Infof("removed %d / %d markers with non-existent faces", removed, n) + } else { + log.Error(err) + } conflicts := 0 - faces, err := query.Faces(true, "") + faces, err := query.Faces(true, false) if err != nil { return err @@ -77,7 +99,11 @@ func (w *Faces) Audit(fix bool) (err error) { } } - log.Infof("%d ambiguous faces clusters", conflicts) + if conflicts == 0 { + log.Infof("found no ambiguous faces clusters") + } else { + log.Infof("%d ambiguous faces clusters", conflicts) + } if markers, err := query.MarkersWithSubjectConflict(); err != nil { log.Error(err) diff --git a/internal/photoprism/faces_cluster.go b/internal/photoprism/faces_cluster.go index f5efbf9b0..5146de8ee 100644 --- a/internal/photoprism/faces_cluster.go +++ b/internal/photoprism/faces_cluster.go @@ -15,18 +15,16 @@ func (w *Faces) Cluster(opt FacesOptions) (added entity.Faces, err error) { return added, fmt.Errorf("facial recognition is disabled") } - minScore := 30 - // Skip clustering if index contains no new face markers, and force option isn't set. if opt.Force { log.Infof("faces: forced clustering") - } else if n := query.CountNewFaceMarkers(minScore); n < opt.SampleThreshold() { + } else if n := query.CountNewFaceMarkers(face.ClusterMinSize, face.ClusterMinScore); n < opt.SampleThreshold() { log.Debugf("faces: skipping clustering") return added, nil } // Fetch unclustered face embeddings. - embeddings, err := query.Embeddings(false, true, minScore) + embeddings, err := query.Embeddings(false, true, face.ClusterMinSize, face.ClusterMinScore) log.Debugf("faces: %d unclustered samples found", len(embeddings)) @@ -76,7 +74,7 @@ func (w *Faces) Cluster(opt FacesOptions) (added entity.Faces, err error) { } else if err := f.Create(); err == nil { added = append(added, *f) log.Debugf("faces: added cluster %s based on %d samples, radius %f", f.ID, f.Samples, f.SampleRadius) - } else if err := f.Updates(entity.Values{"UpdatedAt": entity.Timestamp()}); err != nil { + } else if err := f.Updates(entity.Values{"UpdatedAt": entity.TimeStamp()}); err != nil { log.Errorf("faces: %s", err) } else { log.Debugf("faces: updated cluster %s", f.ID) diff --git a/internal/photoprism/faces_match.go b/internal/photoprism/faces_match.go index cfb14892c..0dae0323f 100644 --- a/internal/photoprism/faces_match.go +++ b/internal/photoprism/faces_match.go @@ -15,37 +15,87 @@ type FacesMatchResult struct { Unknown int64 } +// Add adds result counts. +func (r *FacesMatchResult) Add(result FacesMatchResult) { + r.Updated += result.Updated + r.Recognized += result.Recognized + r.Unknown += result.Unknown +} + // Match matches markers with faces and subjects. func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) { if w.Disabled() { return result, fmt.Errorf("facial recognition is disabled") } - var n int - - matchedBefore := entity.Timestamp() + var unmatchedMarkers int // Skip matching if index contains no new face markers, and force option isn't set. if opt.Force { - log.Infof("faces: forced matching") - } else if n, matchedBefore = query.CountUnmatchedFaceMarkers(); n > 0 { - log.Infof("faces: %d unmatched markers", n) + log.Infof("faces: updating all markers") + } else if unmatchedMarkers = query.CountUnmatchedFaceMarkers(); unmatchedMarkers > 0 { + log.Infof("faces: %d unmatched markers", unmatchedMarkers) } else { - result.Recognized, err = query.MatchFaceMarkers() - return result, err + log.Debugf("faces: no unmatched markers") } - faces, err := query.Faces(false, "") + matchedAt := entity.TimePointer() - if err != nil { - return result, err + if opt.Force || unmatchedMarkers > 0 { + faces, err := query.Faces(false, false) + + if err != nil { + return result, err + } + + if r, err := w.MatchFaces(faces, opt.Force, nil); err != nil { + return result, err + } else { + result.Add(r) + } } + // Find unmatched faces. + if unmatchedFaces, err := query.Faces(false, true); err != nil { + log.Error(err) + } else if len(unmatchedFaces) > 0 { + if r, err := w.MatchFaces(unmatchedFaces, false, matchedAt); err != nil { + return result, err + } else { + result.Add(r) + } + + for _, m := range unmatchedFaces { + if err := m.Matched(); err != nil { + log.Warnf("faces: %s (update match timestamp)", err) + } + } + } + + // Update remaining markers based on previous matches. + if m, err := query.MatchFaceMarkers(); err != nil { + return result, err + } else { + result.Recognized += m + } + + return result, nil +} + +// MatchFaces matches markers against a slice of faces. +func (w *Faces) MatchFaces(faces entity.Faces, force bool, matchedBefore *time.Time) (result FacesMatchResult, err error) { matched := 0 - limit := 100 + limit := 500 + max := query.CountMarkers(entity.MarkerFace) for { - markers, err := query.Markers(limit, 0, entity.MarkerFace, true, false, matchedBefore) + var markers entity.Markers + + if force { + markers, err = query.FaceMarkers(limit, matched) + } else { + markers, err = query.UnmatchedFaceMarkers(limit, 0, matchedBefore) + } if err != nil { return result, err @@ -62,6 +112,11 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) { return result, fmt.Errorf("worker canceled") } + // Skip invalid markers. + if marker.MarkerInvalid || marker.MarkerType != entity.MarkerFace || len(marker.EmbeddingsJSON) == 0 { + continue + } + // Pointer to the matching face. var f *entity.Face @@ -76,10 +131,23 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) { } } + // Marker already has the best matching face? + if !marker.HasFace(f, d) { + // Marker needs a (new) face. + } else { + log.Debugf("faces: marker %d already has the best matching face %s with dist %f", marker.ID, marker.FaceID, marker.FaceDist) + + if err := marker.Matched(); err != nil { + log.Warnf("faces: %s while updating marker %d match timestamp", err, marker.ID) + } + + continue + } + // No matching face? if f == nil { if updated, err := marker.ClearFace(); err != nil { - log.Warnf("faces: %s (clear match)", err) + log.Warnf("faces: %s (clear marker face)", err) } else if updated { result.Updated++ } @@ -88,10 +156,10 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) { } // Assign matching face to marker. - updated, err := marker.SetFace(f) + updated, err := marker.SetFace(f, d) if err != nil { - log.Warnf("faces: %s (match)", err) + log.Warnf("faces: %s while setting a face for marker %d", err, marker.ID) continue } @@ -108,26 +176,12 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) { log.Debugf("faces: matched %d markers", matched) - if matched > query.CountMarkers(entity.MarkerFace) { + if matched > max { break } time.Sleep(50 * time.Millisecond) } - // Update remaining markers based on previous matches. - if m, err := query.MatchFaceMarkers(); err != nil { - return result, err - } else { - result.Recognized += m - } - - // Update face match timestamps. - for _, m := range faces { - if err := m.UpdateMatchTime(); err != nil { - log.Warnf("faces: %s (update match time)", err) - } - } - - return result, nil + return result, err } diff --git a/internal/photoprism/faces_optimize.go b/internal/photoprism/faces_optimize.go index 1e3b621fa..8d1215bef 100644 --- a/internal/photoprism/faces_optimize.go +++ b/internal/photoprism/faces_optimize.go @@ -18,7 +18,7 @@ func (w *Faces) Optimize() (result FacesOptimizeResult, err error) { return result, fmt.Errorf("facial recognition is disabled") } - faces, err := query.Faces(true, entity.SrcManual) + faces, err := query.ManuallyAddedFaces() if err != nil { return result, err diff --git a/internal/photoprism/faces_stats.go b/internal/photoprism/faces_stats.go index 1bb1c6ad7..2f7cc47de 100644 --- a/internal/photoprism/faces_stats.go +++ b/internal/photoprism/faces_stats.go @@ -8,7 +8,7 @@ import ( // Stats shows statistics on face embeddings. func (w *Faces) Stats() (err error) { - if embeddings, err := query.Embeddings(true, false, 0); err != nil { + if embeddings, err := query.Embeddings(true, false, 0, 0); err != nil { return err } else if samples := len(embeddings); samples == 0 { log.Infof("faces: no samples found") @@ -55,7 +55,7 @@ func (w *Faces) Stats() (err error) { log.Infof("faces: max Ø %f < median %f < %f", maxMin, maxMedian, maxMax) } - if faces, err := query.Faces(true, ""); err != nil { + if faces, err := query.Faces(true, false); err != nil { log.Errorf("faces: %s", err) } else if samples := len(faces); samples > 0 { log.Infof("faces: computing distance of faces matching to the same person") diff --git a/internal/query/faces.go b/internal/query/faces.go index 0eaf691c5..fbaa38b1e 100644 --- a/internal/query/faces.go +++ b/internal/query/faces.go @@ -6,30 +6,36 @@ import ( "github.com/photoprism/photoprism/internal/entity" ) -// Faces returns all (known) faces from the index. -func Faces(knownOnly bool, src string) (result entity.Faces, err error) { - stmt := Db() +// Faces returns all (known / unmatched) faces from the index. +func Faces(knownOnly, unmatched bool) (result entity.Faces, err error) { + stmt := Db().Where("face_src <> ?", entity.SrcDefault) - if src == "" { - stmt = stmt.Where("face_src <> ?", entity.SrcDefault) - } else { - stmt = stmt.Where("face_src = ?", src) + if unmatched { + stmt = stmt.Where("matched_at IS NULL") } if knownOnly { - stmt = stmt.Where("subject_uid <> ''").Order("subject_uid, samples DESC") - } else { - stmt = stmt.Order("samples DESC") + stmt = stmt.Where("subject_uid <> ''") } - err = stmt.Find(&result).Error + err = stmt.Order("subject_uid, samples DESC").Find(&result).Error + + return result, err +} + +// ManuallyAddedFaces returns all manually added face clusters. +func ManuallyAddedFaces() (result entity.Faces, err error) { + err = Db(). + Where("face_src = ?", entity.SrcManual). + Where("subject_uid <> ''").Order("subject_uid, samples DESC"). + Find(&result).Error return result, err } // MatchFaceMarkers matches markers with known faces. func MatchFaceMarkers() (affected int64, err error) { - faces, err := Faces(true, "") + faces, err := Faces(true, false) if err != nil { return affected, err @@ -45,10 +51,6 @@ func MatchFaceMarkers() (affected int64, err error) { } else if res.RowsAffected > 0 { affected += res.RowsAffected } - - if err := f.UpdateMatchTime(); err != nil { - return affected, err - } } return affected, nil @@ -72,7 +74,7 @@ func RemoveAutoFaceClusters() (removed int64, err error) { } // CountNewFaceMarkers counts the number of new face markers in the index. -func CountNewFaceMarkers(score int) (n int) { +func CountNewFaceMarkers(size, score int) (n int) { var f entity.Face if err := Db().Where("face_src = ?", entity.SrcAuto). @@ -84,6 +86,10 @@ func CountNewFaceMarkers(score int) (n int) { Where("marker_type = ?", entity.MarkerFace). Where("face_id = '' AND marker_invalid = 0 AND embeddings_json <> ''") + if size > 0 { + q = q.Where("size >= ?", size) + } + if score > 0 { q = q.Where("score >= ?", score) } @@ -99,6 +105,21 @@ func CountNewFaceMarkers(score int) (n int) { return n } +// RemoveUnusedFaces removes unused faces from the index. +func RemoveUnusedFaces(faceIds []string) (removed int64, err error) { + // Remove invalid face IDs. + if res := Db(). + Where("id IN (?)", faceIds). + Where(fmt.Sprintf("id NOT IN (SELECT face_id FROM %s)", entity.Marker{}.TableName())). + Delete(&entity.Face{}); res.Error != nil { + return removed, res.Error + } else { + removed += res.RowsAffected + } + + return removed, nil +} + // MergeFaces returns a new face that replaces multiple others. func MergeFaces(merge entity.Faces) (merged *entity.Face, err error) { if len(merge) < 2 { @@ -111,22 +132,15 @@ func MergeFaces(merge entity.Faces) (merged *entity.Face, err error) { return merged, fmt.Errorf("merged face must not be nil") } else if err := merged.Create(); err != nil { return merged, err - } - - // Update marker matches. - if err := Db().Model(&entity.Marker{}).Where("face_id IN (?)", merge.IDs()). - Updates(entity.Values{"face_id": merged.ID, "subject_uid": merged.SubjectUID}).Error; err != nil { + } else if err := merged.MatchMarkers(append(merge.IDs(), "")); err != nil { return merged, err } - // Delete merged faces. - if err := Db().Where("id IN (?) AND id <> ?", merge.IDs(), merged.ID).Delete(&entity.Face{}).Error; err != nil { - return merged, err - } - - // Find and reference additional matching markers. - if err := merged.MatchMarkers(); err != nil { - return merged, err + // RemoveUnusedFaces removes unused faces from the index. + if removed, err := RemoveUnusedFaces(merge.IDs()); err != nil { + log.Errorf("faces: %s", err) + } else { + log.Debugf("faces: removed %d unused faces", removed) } return merged, err diff --git a/internal/query/faces_test.go b/internal/query/faces_test.go index a1a49196c..923160846 100644 --- a/internal/query/faces_test.go +++ b/internal/query/faces_test.go @@ -9,17 +9,49 @@ import ( ) func TestFaces(t *testing.T) { - results, err := Faces(true, "") + t.Run("known", func(t *testing.T) { + results, err := Faces(true, false) - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) + } - assert.GreaterOrEqual(t, len(results), 1) + assert.GreaterOrEqual(t, len(results), 1) - for _, val := range results { - assert.IsType(t, entity.Face{}, val) - } + for _, val := range results { + assert.IsType(t, entity.Face{}, val) + } + }) + + t.Run("unmatched", func(t *testing.T) { + results, err := Faces(false, true) + + if err != nil { + t.Fatal(err) + } + + assert.GreaterOrEqual(t, len(results), 1) + + for _, val := range results { + assert.IsType(t, entity.Face{}, val) + } + }) +} + +func TestManuallyAddedFaces(t *testing.T) { + t.Run("success", func(t *testing.T) { + results, err := ManuallyAddedFaces() + + if err != nil { + t.Fatal(err) + } + + assert.GreaterOrEqual(t, len(results), 1) + + for _, val := range results { + assert.IsType(t, entity.Face{}, val) + } + }) } func TestMatchFaceMarkers(t *testing.T) { @@ -65,6 +97,16 @@ func TestRemoveAnonymousFaceClusters(t *testing.T) { } func TestCountNewFaceMarkers(t *testing.T) { - n := CountNewFaceMarkers(1) - assert.GreaterOrEqual(t, n, 1) + t.Run("all", func(t *testing.T) { + assert.GreaterOrEqual(t, CountNewFaceMarkers(0, 0), 1) + }) + t.Run("score 10", func(t *testing.T) { + assert.GreaterOrEqual(t, CountNewFaceMarkers(0, 10), 1) + }) + t.Run("size 160", func(t *testing.T) { + assert.GreaterOrEqual(t, CountNewFaceMarkers(160, 0), 1) + }) + t.Run("score 50 and size 160", func(t *testing.T) { + assert.GreaterOrEqual(t, CountNewFaceMarkers(160, 50), 1) + }) } diff --git a/internal/query/markers.go b/internal/query/markers.go index 53ee5c834..c1abbcf92 100644 --- a/internal/query/markers.go +++ b/internal/query/markers.go @@ -44,8 +44,38 @@ func Markers(limit, offset int, markerType string, embeddings, subjects bool, ma return result, err } +// UnmatchedFaceMarkers finds all currently unmatched face markers. +func UnmatchedFaceMarkers(limit, offset int, matchedBefore *time.Time) (result entity.Markers, err error) { + db := Db(). + Where("marker_type = ?", entity.MarkerFace). + Where("marker_invalid = 0"). + Where("embeddings_json <> ''") + + if matchedBefore == nil { + db = db.Where("matched_at IS NULL") + } else if !matchedBefore.IsZero() { + db = db.Where("matched_at IS NULL OR matched_at < ?", matchedBefore) + } + + db = db.Order("matched_at, id").Limit(limit).Offset(offset) + + err = db.Find(&result).Error + + return result, err +} + +// FaceMarkers returns all face markers sorted by id. +func FaceMarkers(limit, offset int) (result entity.Markers, err error) { + err = Db(). + Where("marker_type = ?", entity.MarkerFace). + Order("id").Limit(limit).Offset(offset). + Find(&result).Error + + return result, err +} + // Embeddings returns existing face embeddings. -func Embeddings(single, unclustered bool, score int) (result entity.Embeddings, err error) { +func Embeddings(single, unclustered bool, size, score int) (result entity.Embeddings, err error) { var col []string stmt := Db(). @@ -55,6 +85,10 @@ func Embeddings(single, unclustered bool, score int) (result entity.Embeddings, Where("embeddings_json <> ''"). Order("id") + if size > 0 { + stmt = stmt.Where("size >= ?", size) + } + if score > 0 { stmt = stmt.Where("score >= ?", score) } @@ -82,44 +116,63 @@ func Embeddings(single, unclustered bool, score int) (result entity.Embeddings, return result, nil } -// RemoveInvalidMarkerReferences deletes invalid reference IDs from the markers table. +// RemoveInvalidMarkerReferences removes face and subject references from invalid markers. func RemoveInvalidMarkerReferences() (removed int64, err error) { - // Remove subject and face relationships for invalid markers. - if res := Db(). + res := Db(). Model(&entity.Marker{}). Where("marker_invalid = 1 AND (subject_uid <> '' OR face_id <> '')"). - UpdateColumns(entity.Values{"subject_uid": "", "face_id": ""}); res.Error != nil { - return removed, res.Error - } else { - removed += res.RowsAffected - } + UpdateColumns(entity.Values{"subject_uid": "", "face_id": "", "face_dist": -1.0, "matched_at": nil}) - // Remove invalid face IDs. - if res := Db(). + return res.RowsAffected, res.Error +} + +// RemoveNonExistentMarkerFaces removes non-existent face IDs from the markers table. +func RemoveNonExistentMarkerFaces() (removed int64, err error) { + + res := Db(). Model(&entity.Marker{}). Where("marker_type = ?", entity.MarkerFace). Where(fmt.Sprintf("face_id <> '' AND face_id NOT IN (SELECT id FROM %s)", entity.Face{}.TableName())). - UpdateColumns(entity.Values{"face_id": ""}); res.Error != nil { - return removed, res.Error - } else { - removed += res.RowsAffected - } + UpdateColumns(entity.Values{"face_id": "", "face_dist": -1.0, "matched_at": nil}) - // Remove invalid subject UIDs. - if res := Db(). + return res.RowsAffected, res.Error +} + +// RemoveNonExistentMarkerSubjects removes non-existent subject UIDs from the markers table. +func RemoveNonExistentMarkerSubjects() (removed int64, err error) { + res := Db(). Model(&entity.Marker{}). Where(fmt.Sprintf("subject_uid <> '' AND subject_uid NOT IN (SELECT subject_uid FROM %s)", entity.Subject{}.TableName())). - UpdateColumns(entity.Values{"subject_uid": ""}); res.Error != nil { - return removed, res.Error + UpdateColumns(entity.Values{"subject_uid": "", "matched_at": nil}) + + return res.RowsAffected, res.Error +} + +// FixMarkerReferences repairs invalid or non-existent references in the markers table. +func FixMarkerReferences() (removed int64, err error) { + if r, err := RemoveInvalidMarkerReferences(); err != nil { + return removed, err } else { - removed += res.RowsAffected + removed += r + } + + if r, err := RemoveNonExistentMarkerFaces(); err != nil { + return removed, err + } else { + removed += r + } + + if r, err := RemoveNonExistentMarkerSubjects(); err != nil { + return removed, err + } else { + removed += r } return removed, nil } -// MarkersWithInvalidReferences finds markers with invalid references. -func MarkersWithInvalidReferences() (faces entity.Markers, subjects entity.Markers, err error) { +// MarkersWithNonExistentReferences finds markers with non-existent face or subject references. +func MarkersWithNonExistentReferences() (faces entity.Markers, subjects entity.Markers, err error) { // Find markers with invalid face IDs. if res := Db(). Where("marker_type = ?", entity.MarkerFace). @@ -152,36 +205,22 @@ func MarkersWithSubjectConflict() (results entity.Markers, err error) { func ResetFaceMarkerMatches() (removed int64, err error) { res := Db().Model(&entity.Marker{}). Where("subject_src <> ? AND marker_type = ?", entity.SrcManual, entity.MarkerFace). - UpdateColumns(entity.Values{"subject_uid": "", "subject_src": "", "face_id": "", "matched_at": nil}) + UpdateColumns(entity.Values{"subject_uid": "", "subject_src": "", "face_id": "", "face_dist": -1.0, "matched_at": nil}) return res.RowsAffected, res.Error } // CountUnmatchedFaceMarkers counts the number of unmatched face markers in the index. -func CountUnmatchedFaceMarkers() (n int, matchedBefore time.Time) { - var f entity.Face - - if err := Db().Where("face_src <> ?", entity.SrcDefault). - Order("updated_at DESC").Limit(1).Take(&f).Error; err != nil || f.UpdatedAt.IsZero() { - return 0, matchedBefore - } - - matchedBefore = time.Now().UTC().Round(time.Second).Add(-2 * time.Hour) - - if f.UpdatedAt.Before(matchedBefore) { - matchedBefore = f.UpdatedAt.Add(time.Second) - } - +func CountUnmatchedFaceMarkers() (n int) { q := Db().Model(&entity.Markers{}). - Where("marker_type = ?", entity.MarkerFace). - Where("face_id = '' AND subject_src = '' AND marker_invalid = 0 AND embeddings_json <> ''"). - Where("matched_at IS NULL OR matched_at < ?", matchedBefore) + Where("matched_at IS NULL AND marker_invalid = 0 AND embeddings_json <> ''"). + Where("marker_type = ?", entity.MarkerFace) if err := q.Count(&n).Error; err != nil { log.Errorf("faces: %s (count unmatched markers)", err) } - return n, matchedBefore + return n } // CountMarkers counts the number of face markers in the index. diff --git a/internal/query/markers_test.go b/internal/query/markers_test.go index 8f275aef3..4943a60c9 100644 --- a/internal/query/markers_test.go +++ b/internal/query/markers_test.go @@ -10,7 +10,7 @@ import ( func TestMarkers(t *testing.T) { t.Run("find umatched", func(t *testing.T) { - results, err := Markers(3, 0, entity.MarkerFace, false, false, entity.Timestamp()) + results, err := Markers(3, 0, entity.MarkerFace, false, false, entity.TimeStamp()) if err != nil { t.Fatal(err) @@ -59,29 +59,111 @@ func TestMarkers(t *testing.T) { }) } +func TestUnmatchedFaceMarkers(t *testing.T) { + t.Run("all", func(t *testing.T) { + results, err := UnmatchedFaceMarkers(3, 0, nil) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 3, len(results)) + }) + t.Run("before", func(t *testing.T) { + results, err := UnmatchedFaceMarkers(3, 0, entity.TimePointer()) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 3, len(results)) + }) +} + +func TestFaceMarkers(t *testing.T) { + t.Run("all", func(t *testing.T) { + results, err := FaceMarkers(3, 0) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 3, len(results)) + }) +} + func TestEmbeddings(t *testing.T) { - results, err := Embeddings(false, false, 0) + t.Run("all", func(t *testing.T) { + results, err := Embeddings(false, false, 0, 0) - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) + } - assert.GreaterOrEqual(t, len(results), 1) + assert.GreaterOrEqual(t, len(results), 1) - for _, val := range results { - assert.IsType(t, entity.Embedding{}, val) - } + for _, val := range results { + assert.IsType(t, entity.Embedding{}, val) + } + }) + t.Run("size", func(t *testing.T) { + results, err := Embeddings(false, false, 230, 0) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, len(results), 1) + + for _, val := range results { + assert.IsType(t, entity.Embedding{}, val) + } + }) + t.Run("score", func(t *testing.T) { + results, err := Embeddings(false, false, 0, 50) + + if err != nil { + t.Fatal(err) + } + + assert.GreaterOrEqual(t, len(results), 1) + + for _, val := range results { + assert.IsType(t, entity.Embedding{}, val) + } + }) } func TestRemoveInvalidMarkerReferences(t *testing.T) { affected, err := RemoveInvalidMarkerReferences() + assert.NoError(t, err) + assert.GreaterOrEqual(t, affected, int64(0)) +} + +func TestRemoveNonExistentMarkerFaces(t *testing.T) { + affected, err := RemoveNonExistentMarkerFaces() + assert.NoError(t, err) assert.GreaterOrEqual(t, affected, int64(1)) } -func TestMarkersWithInvalidReferences(t *testing.T) { - f, s, err := MarkersWithInvalidReferences() +func TestRemoveNonExistentMarkerSubjects(t *testing.T) { + affected, err := RemoveNonExistentMarkerSubjects() + + assert.NoError(t, err) + assert.GreaterOrEqual(t, affected, int64(1)) +} + +func TestFixMarkerReferences(t *testing.T) { + affected, err := FixMarkerReferences() + + assert.NoError(t, err) + assert.GreaterOrEqual(t, affected, int64(0)) +} + +func TestMarkersWithNonExistentReferences(t *testing.T) { + f, s, err := MarkersWithNonExistentReferences() assert.NoError(t, err) @@ -98,10 +180,9 @@ func TestMarkersWithSubjectConflict(t *testing.T) { } func TestCountUnmatchedFaceMarkers(t *testing.T) { - n, threshold := CountUnmatchedFaceMarkers() + n := CountUnmatchedFaceMarkers() - assert.False(t, threshold.IsZero()) - assert.GreaterOrEqual(t, n, 0) + assert.GreaterOrEqual(t, n, 1) } func TestCountMarkers(t *testing.T) { diff --git a/internal/query/subjects.go b/internal/query/subjects.go index adee096d6..9ad12dbe6 100644 --- a/internal/query/subjects.go +++ b/internal/query/subjects.go @@ -48,8 +48,8 @@ func RemoveDanglingMarkerSubjects() (removed int64, err error) { return res.RowsAffected, res.Error } -// AddFaceMarkerSubjects adds and references known marker subjects. -func AddFaceMarkerSubjects() (affected int64, err error) { +// CreateMarkerSubjects adds and references known marker subjects. +func CreateMarkerSubjects() (affected int64, err error) { var markers entity.Markers if err := Db(). diff --git a/internal/query/subjects_test.go b/internal/query/subjects_test.go index 344083037..5e2372e04 100644 --- a/internal/query/subjects_test.go +++ b/internal/query/subjects_test.go @@ -46,8 +46,8 @@ func TestRemoveDanglingMarkerSubjects(t *testing.T) { assert.Equal(t, int64(1), affected) } -func TestAddFaceMarkerSubjects(t *testing.T) { - affected, err := AddFaceMarkerSubjects() +func TestCreateMarkerSubjects(t *testing.T) { + affected, err := CreateMarkerSubjects() assert.NoError(t, err) assert.GreaterOrEqual(t, affected, int64(2))