beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

People: Improve face clustering #22

Browse source 
Work in progress.
This commit is contained in:
Michael Mayer 2021-08-12 12:05:10 +02:00
parent 1fc4ef123b
commit d767e50b37
13 changed files with 108 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
internal/config/test.go
View file

@ -57,6 +57,7 @@ func NewTestOptions() *Options {
Copyright: "(c) 2018-2021 Michael Mayer",
Debug: true,
Public: true,
Experimental: true,
ReadOnly: false,
DetectNSFW: true,
UploadNSFW: false,

15
internal/entity/marker.go
View file

@ -21,11 +21,12 @@ const (
type Marker struct {
ID uint `gorm:"primary_key" json:"ID" yaml:"-"`
FileID uint `gorm:"index;" json:"-" yaml:"-"`
Ref string `gorm:"type:VARBINARY(42);index;" json:"Ref" yaml:"Ref,omitempty"`
FaceID string `gorm:"type:VARBINARY(42);index;" json:"FaceID" yaml:"FaceID,omitempty"`
RefUID string `gorm:"type:VARBINARY(42);index:idx_markers_uid_type;" json:"RefUID" yaml:"RefUID,omitempty"`
RefSrc string `gorm:"type:VARBINARY(8);default:'';" json:"RefSrc" yaml:"RefSrc,omitempty"`
MarkerType string `gorm:"type:VARBINARY(8);index:idx_markers_uid_type;default:'';" json:"Type" yaml:"Type"`
MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"`
MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"`
MarkerScore int `gorm:"type:SMALLINT" json:"Score" yaml:"Score"`
MarkerScore int `gorm:"type:SMALLINT" json:"Score" yaml:"Score,omitempty"`
MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"`
MarkerLabel string `gorm:"type:VARCHAR(255);" json:"Label" yaml:"Label,omitempty"`
MarkerMeta string `gorm:"type:LONGTEXT;" json:"Meta" yaml:"Meta,omitempty"`
@ -50,7 +51,7 @@ func (Marker) TableName() string {
func NewMarker(fileUID uint, refUID, markerSrc, markerType string, x, y, w, h float32) *Marker {
m := &Marker{
FileID: fileUID,
Ref: refUID,
RefUID: refUID,
MarkerSrc: markerSrc,
MarkerType: markerType,
X: x,
@ -77,12 +78,12 @@ func NewFaceMarker(f face.Face, fileID uint, refUID string) *Marker {
// Updates multiple columns in the database.
func (m *Marker) Updates(values interface{}) error {
return UnscopedDb().Model(m).UpdateColumns(values).Error
return UnscopedDb().Model(m).Updates(values).Error
}
// Update updates a column in the database.
func (m *Marker) Update(attr string, value interface{}) error {
return UnscopedDb().Model(m).UpdateColumn(attr, value).Error
return UnscopedDb().Model(m).Update(attr, value).Error
}
// SaveForm updates the entity using form data and stores it in the database.
@ -162,7 +163,7 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) {
"MarkerScore": m.MarkerScore,
"MarkerMeta": m.MarkerMeta,
"Embeddings": m.Embeddings,
"Ref": m.Ref,
"RefUID": m.RefUID,
})
log.Debugf("faces: updated existing marker %d for file %d", result.ID, result.FileID)

10
internal/entity/marker_fixtures.go
View file

@ -21,7 +21,7 @@ func (m MarkerMap) Pointer(name string) *Marker {
var MarkerFixtures = MarkerMap{
"1000003-1": Marker{
FileID: 1000003,
Ref: "lt9k3pw1wowuy3c3",
RefUID: "lt9k3pw1wowuy3c3",
MarkerSrc: SrcImage,
MarkerType: MarkerLabel,
X: 0.308333,
@ -31,7 +31,7 @@ var MarkerFixtures = MarkerMap{
},
"1000003-2": Marker{
FileID: 1000003,
Ref: "",
RefUID: "",
MarkerLabel: "Unknown",
MarkerSrc: SrcImage,
MarkerType: MarkerLabel,
@ -42,7 +42,7 @@ var MarkerFixtures = MarkerMap{
},
"1000003-3": Marker{
FileID: 1000003,
Ref: "",
RefUID: "",
MarkerSrc: SrcImage,
MarkerType: MarkerLabel,
MarkerLabel: "Center",
@ -53,7 +53,7 @@ var MarkerFixtures = MarkerMap{
},
"1000003-4": Marker{
FileID: 1000003,
Ref: "",
RefUID: "",
MarkerSrc: SrcImage,
MarkerType: MarkerFace,
MarkerLabel: "Jens Mander",
@ -66,7 +66,7 @@ var MarkerFixtures = MarkerMap{
},
"1000003-5": Marker{
FileID: 1000003,
Ref: "",
RefUID: "",
MarkerSrc: SrcImage,
MarkerType: MarkerFace,
MarkerLabel: "Corn McCornface",

4
internal/entity/marker_test.go
View file

@ -15,7 +15,7 @@ func TestNewMarker(t *testing.T) {
m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
assert.IsType(t, &Marker{}, m)
assert.Equal(t, uint(1000000), m.FileID)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.Ref)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID)
assert.Equal(t, SrcImage, m.MarkerSrc)
assert.Equal(t, MarkerLabel, m.MarkerType)
}
@ -25,7 +25,7 @@ func TestUpdateOrCreateMarker(t *testing.T) {
m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
assert.IsType(t, &Marker{}, m)
assert.Equal(t, uint(1000000), m.FileID)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.Ref)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID)
assert.Equal(t, SrcImage, m.MarkerSrc)
assert.Equal(t, MarkerLabel, m.MarkerType)

46
internal/entity/person_face.go
View file

@ -2,7 +2,7 @@ package entity
import (
"crypto/sha1"
"fmt"
"encoding/base32"
"time"
)
@ -10,14 +10,12 @@ type PeopleFaces []PersonFace
// PersonFace represents the face of a Person.
type PersonFace struct {
ID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"ID" yaml:"ID"`
PersonUID string `gorm:"type:VARBINARY(42);index;" json:"PersonUID" yaml:"PersonUID"`
FaceSrc string `gorm:"type:VARBINARY(8);" json:"Src" yaml:"Src"`
Embedding string `gorm:"type:LONGTEXT;" json:"Embedding" yaml:"Embedding,omitempty"`
PhotoCount int `gorm:"default:0" json:"PhotoCount" yaml:"-"`
CreatedAt time.Time `json:"CreatedAt" yaml:"-"`
UpdatedAt time.Time `json:"UpdatedAt" yaml:"-"`
DeletedAt *time.Time `sql:"index" json:"DeletedAt,omitempty" yaml:"-"`
ID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"ID" yaml:"ID"`
PersonUID string `gorm:"type:VARBINARY(42);index;" json:"PersonUID" yaml:"PersonUID"`
Embedding string `gorm:"type:LONGTEXT;" json:"Embedding" yaml:"Embedding,omitempty"`
CreatedAt time.Time `json:"CreatedAt" yaml:"CreatedAt,omitempty"`
UpdatedAt time.Time `json:"UpdatedAt" yaml:"UpdatedAt,omitempty"`
DeletedAt *time.Time `sql:"index" json:"DeletedAt,omitempty" yaml:"-"`
}
// TableName returns the entity database table name.
@ -25,20 +23,17 @@ func (PersonFace) TableName() string {
return "people_faces_dev"
}
/*
// BeforeCreate creates a random UID if needed before inserting a new row to the database.
func (m *PersonFace) BeforeCreate(scope *gorm.Scope) error {
return scope.SetColumn("ID")
}*/
// NewPersonFace returns a new face.
func NewPersonFace(personUID, faceSrc, embedding string, photoCount int) *PersonFace {
func NewPersonFace(personUID, embedding string) *PersonFace {
timeStamp := Timestamp()
s := sha1.Sum([]byte(embedding))
result := &PersonFace{
ID: fmt.Sprintf("%x", sha1.Sum([]byte(embedding))),
PersonUID: personUID,
FaceSrc: faceSrc,
Embedding: embedding,
PhotoCount: photoCount,
ID: base32.StdEncoding.EncodeToString(s[:]),
PersonUID: personUID,
Embedding: embedding,
CreatedAt: timeStamp,
UpdatedAt: timeStamp,
}
return result
@ -54,7 +49,7 @@ func (m *PersonFace) Save() error {
peopleMutex.Lock()
defer peopleMutex.Unlock()
return Db().Save(m).Error
return Save(m, "ID")
}
// Create inserts the face to the database.
@ -86,5 +81,10 @@ func (m *PersonFace) Restore() error {
// Update a face property in the database.
func (m *PersonFace) Update(attr string, value interface{}) error {
return UnscopedDb().Model(m).UpdateColumn(attr, value).Error
return UnscopedDb().Model(m).Update(attr, value).Error
}
// Updates face properties in the database.
func (m *PersonFace) Updates(values interface{}) error {
return UnscopedDb().Model(m).Updates(values).Error
}

4
internal/entity/src.go
View file

@ -9,6 +9,7 @@ const (
SrcAuto = ""
SrcManual = "manual"
SrcEstimate = "estimate"
SrcPeople = "people"
SrcName = "name"
SrcMeta = "meta"
SrcXmp = "xmp"
@ -18,10 +19,11 @@ const (
SrcLocation = classify.SrcLocation
)
// Data source priorities.
// SrcPriority maps source priorities.
var SrcPriority = Priorities{
SrcAuto: 1,
SrcEstimate: 2,
SrcPeople: 2,
SrcName: 4,
SrcYaml: 8,
SrcLocation: 8,

4
internal/entity/val.go Normal file
View file

@ -0,0 +1,4 @@
package entity
// Val is a shortcut for map[string]interface{}
type Val map[string]interface{}

12
internal/face/net.go
View file

@ -114,9 +114,9 @@ func (t *Net) getFaceCrop(fileName, fileHash string, f Point) (img image.Image,
if !fs.FileExists(cacheFile) {
// Do nothing.
} else if img, err := imaging.Open(cacheFile); err != nil {
log.Errorf("faces: failed loading cached face crop %s", filepath.Base(cacheFile))
log.Errorf("faces: failed loading cached crop %s", filepath.Base(cacheFile))
} else {
log.Debugf("faces: found cached face crop %s", filepath.Base(cacheFile))
log.Debugf("faces: found cached crop %s", filepath.Base(cacheFile))
return img, nil
}
@ -133,9 +133,9 @@ func (t *Net) getFaceCrop(fileName, fileHash string, f Point) (img image.Image,
img = imaging.Fill(img, 160, 160, imaging.Center, imaging.Lanczos)
if err := imaging.Save(img, cacheFile); err != nil {
log.Errorf("faces: failed caching face crop %s", filepath.Base(cacheFile))
log.Errorf("faces: failed caching crop %s", filepath.Base(cacheFile))
} else {
log.Debugf("faces: saved face crop %s", filepath.Base(cacheFile))
log.Debugf("faces: saved crop %s", filepath.Base(cacheFile))
}
return img, nil
@ -147,9 +147,11 @@ func (t *Net) getEmbeddings(img image.Image) [][]float32 {
if err != nil {
log.Errorf("faces: failed to convert image to tensor: %v", err)
}
// TODO: prewhiten image as in facenet
trainPhaseBoolTensor, err := tf.NewTensor(false)
output, err := t.model.Session.Run(
map[tf.Output]*tf.Tensor{
t.model.Graph.Operation("input").Output(0): tensor,
@ -168,8 +170,8 @@ func (t *Net) getEmbeddings(img image.Image) [][]float32 {
log.Errorf("faces: inference failed, no output")
} else {
return output[0].Value().([][]float32)
// embeddings = append(embeddings, output[0].Value().([][]float32)[0])
}
return nil
}

2
internal/form/marker.go
View file

@ -4,7 +4,7 @@ import "github.com/ulule/deepcopier"
// Marker represents an image marker edit form.
type Marker struct {
Ref string `json:"Ref"`
RefUID string `json:"RefUID"`
RefSrc string `json:"RefSrc"`
MarkerSrc string `json:"Src"`
MarkerType string `json:"Type"`

6
internal/form/marker_test.go
View file

@ -9,7 +9,7 @@ import (
func TestNewMarker(t *testing.T) {
t.Run("success", func(t *testing.T) {
var m = struct {
Ref string
RefUID string
RefSrc string
MarkerSrc string
MarkerType string
@ -17,7 +17,7 @@ func TestNewMarker(t *testing.T) {
MarkerInvalid bool
MarkerLabel string
}{
Ref: "3h59wvth837b5vyiub35",
RefUID: "3h59wvth837b5vyiub35",
RefSrc: "meta",
MarkerSrc: "image",
MarkerType: "Face",
@ -32,7 +32,7 @@ func TestNewMarker(t *testing.T) {
t.Fatal(err)
}
assert.Equal(t, "3h59wvth837b5vyiub35", f.Ref)
assert.Equal(t, "3h59wvth837b5vyiub35", f.RefUID)
assert.Equal(t, "meta", f.RefSrc)
assert.Equal(t, "image", f.MarkerSrc)
assert.Equal(t, "Face", f.MarkerType)

75
internal/photoprism/people.go
View file

@ -14,7 +14,7 @@ import (
"github.com/mpraski/clusters"
)
// People represents a worker that clusters face embeddings to search for individual people.
// People represents a worker for face clustering and recognition.
type People struct {
conf *config.Config
}
@ -28,7 +28,7 @@ func NewPeople(conf *config.Config) *People {
return instance
}
// Start clusters face embeddings to search for individual people.
// Start face clustering and recognition.
func (m *People) Start() (err error) {
defer func() {
if r := recover(); r != nil {
@ -37,6 +37,12 @@ func (m *People) Start() (err error) {
}
}()
if !m.conf.Experimental() {
return fmt.Errorf("people: experimental features disabled")
} else if !m.conf.Settings().Features.People {
return fmt.Errorf("people: disabled in settings")
}
if err := mutex.MainWorker.Start(); err != nil {
return err
}
@ -56,7 +62,7 @@ func (m *People) Start() (err error) {
// see https://fse.studenttheses.ub.rug.nl/18064/1/Report_research_internship.pdf
c, e := clusters.DBSCAN(1, 0.42, 1, clusters.EuclideanDistance)
c, e := clusters.DBSCAN(1, 0.42, m.conf.Workers(), clusters.EuclideanDistance)
if e != nil {
return e
@ -68,7 +74,7 @@ func (m *People) Start() (err error) {
sizes := c.Sizes()
log.Infof("people: found %d faces from %d people", len(embeddings), len(sizes))
log.Infof("people: found %d embeddings, %d clusters", len(embeddings), len(sizes))
faceClusters := make([]entity.Embeddings, len(sizes))
@ -86,17 +92,29 @@ func (m *People) Start() (err error) {
faceClusters[number-1] = append(faceClusters[number-1], embeddings[index])
}
addedFaces := 0
recognized := 0
markersUpdated := 0
updateErrors := 0
for _, clusterEmb := range faceClusters {
if emb, err := json.Marshal(entity.EmbeddingsMidpoint(clusterEmb)); err != nil {
updateErrors++
log.Errorf("people: %s", err)
} else if f := entity.NewPersonFace("", entity.SrcImage, string(emb), len(clusterEmb)); f == nil {
} else if f := entity.NewPersonFace("", string(emb)); f == nil {
updateErrors++
log.Errorf("people: face should not be nil - bug?")
} else if err := f.Save(); err != nil {
log.Errorf("people: %s while saving face", err)
} else if err := f.Create(); err == nil {
addedFaces++
log.Tracef("people: added face %s", f.ID)
} else if err := f.Updates(entity.Val{"UpdatedAt": entity.Timestamp()}); err != nil {
updateErrors++
log.Errorf("people: %s", err)
}
}
if err := query.PurgeUnknownFaces(); err != nil {
updateErrors++
log.Errorf("people: %s", err)
}
@ -106,25 +124,22 @@ func (m *People) Start() (err error) {
return err
}
faceMap := make(map[string]entity.Embedding)
uidMap := make(map[string]string, len(peopleFaces))
faceMap := make(map[string]entity.Embedding, len(peopleFaces))
for _, f := range peopleFaces {
var id string
faceMap[f.ID] = f.UnmarshalEmbedding()
if f.PersonUID != "" {
id = f.PersonUID
} else {
id = f.ID
uidMap[f.ID] = f.PersonUID
}
faceMap[id] = f.UnmarshalEmbedding()
}
limit := 500
offset := 0
for {
markers, err := query.Markers(limit, offset, entity.MarkerFace, true, false)
markers, err := query.Markers(limit, offset, entity.MarkerFace, true, true)
if err != nil {
return err
@ -139,30 +154,32 @@ func (m *People) Start() (err error) {
return fmt.Errorf("people: worker canceled")
}
if _, ok := faceMap[marker.Ref]; ok {
continue
}
var ref string
var dist float64
var faceId string
var faceDist float64
for _, e1 := range marker.UnmarshalEmbeddings() {
for id, e2 := range faceMap {
if d := clusters.EuclideanDistance(e1, e2); ref == "" || d < dist {
ref = id
dist = d
if d := clusters.EuclideanDistance(e1, e2); faceId == "" || d < faceDist {
faceId = id
faceDist = d
}
}
}
if marker.Ref == ref {
if marker.RefUID != "" && marker.RefUID == uidMap[faceId] {
continue
}
if err := marker.Update("Ref", ref); err != nil {
log.Errorf("people: %s while saving marker", err)
if refUID := uidMap[faceId]; refUID != "" {
if err := marker.Updates(entity.Val{"RefUID": refUID, "RefSrc": entity.SrcPeople, "FaceID": ""}); err != nil {
log.Errorf("people: %s while updating person uid", err)
} else {
recognized++
}
} else if err := marker.Updates(entity.Val{"FaceID": faceId}); err != nil {
log.Errorf("people: %s while updating marker face id", err)
} else {
log.Debugf("people: marker %d ref %s", marker.ID, ref)
markersUpdated++
}
}
@ -171,6 +188,8 @@ func (m *People) Start() (err error) {
time.Sleep(50 * time.Millisecond)
}
log.Infof("people: %d faces added, %d recognized, %d markers updated, %d errors", addedFaces, recognized, markersUpdated, updateErrors)
return nil
}

6
internal/query/markers.go
View file

@ -15,7 +15,7 @@ func MarkerByID(id uint) (marker entity.Marker, err error) {
}
// Markers finds a list of file markers filtered by type, embeddings, and sorted by id.
func Markers(limit, offset int, markerType string, embeddings, noRef bool) (result entity.Markers, err error) {
func Markers(limit, offset int, markerType string, embeddings, unmatched bool) (result entity.Markers, err error) {
stmt := Db()
if markerType != "" {
@ -26,8 +26,8 @@ func Markers(limit, offset int, markerType string, embeddings, noRef bool) (resu
stmt = stmt.Where("embeddings <> ''")
}
if noRef {
stmt = stmt.Where("ref = ''")
if unmatched {
stmt = stmt.Where("ref_uid = ''")
}
stmt = stmt.Order("id").Limit(limit).Offset(offset)

2
internal/query/people.go
View file

@ -32,5 +32,5 @@ func PeopleFaces() (result entity.PeopleFaces, err error) {
func PurgeUnknownFaces() error {
return UnscopedDb().Delete(
entity.PersonFace{},
"face_src = ? AND person_uid = '' AND updated_at < ?", entity.SrcImage, entity.Yesterday()).Error
"person_uid = '' AND updated_at < ?", entity.Yesterday()).Error
}