beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Covers: Improve performance of update queries #383

Browse source
This commit is contained in:
Michael Mayer 2021-10-01 00:05:49 +02:00
parent 0199cab12d
commit e50ede6368
15 changed files with 199 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frontend/src/pages/people/faces.vue
View file

@ -116,7 +116,7 @@
</v-card>
</v-flex>
</v-layout>
<div class="text-xs-center my-2">
<div class="text-xs-center mt-3 mb-2">
<v-btn
color="secondary" round
:to="{name: 'all', query: { q: 'face:new' }}"

8
internal/commands/index_test.go
View file

@ -46,10 +46,10 @@ func TestIndexCommand(t *testing.T) {
assert.Contains(t, output, "indexing originals")
assert.Contains(t, output, "classify: loading labels")
assert.Contains(t, output, "index: no .ppignore file found")
assert.Contains(t, output, "purge: searching index for unassigned primary files")
assert.Contains(t, output, "purge: searching index for hidden media files")
assert.Contains(t, output, "purge: updating photo counts")
assert.Contains(t, output, "purge: updating preview images")
assert.Contains(t, output, "searching index for unassigned primary files")
assert.Contains(t, output, "searching index for hidden media files")
assert.Contains(t, output, "updating photo counts")
assert.Contains(t, output, "updating preview thumbs")
assert.Contains(t, output, "indexed")
assert.Contains(t, output, "files in")
} else {

4
internal/commands/users.go
View file

@ -6,6 +6,8 @@ import (
"fmt"
"strings"
"github.com/dustin/go-humanize/english"
"github.com/manifoldco/promptui"
"github.com/urfave/cli"
@ -205,7 +207,7 @@ func usersDeleteAction(ctx *cli.Context) error {
func usersListAction(ctx *cli.Context) error {
return callWithDependencies(ctx, func(conf *config.Config) error {
users := query.RegisteredUsers()
log.Infof("found %d users", len(users))
log.Infof("found %s", english.Plural(len(users), "user", "users"))
fmt.Printf("%-4s %-16s %-16s %-16s\n", "ID", "LOGIN", "NAME", "EMAIL")

8
internal/entity/file.go
View file

@ -7,6 +7,8 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize/english"
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/ulule/deepcopier"
@ -259,10 +261,8 @@ func (m *File) ReplaceHash(newHash string) error {
if res := UnscopedDb().Model(entity).Where("thumb = ?", oldHash).UpdateColumn("thumb", newHash); res.Error != nil {
return res.Error
} else if res.RowsAffected == 1 {
log.Infof("%s: updated %d preview [%s]", name, res.RowsAffected, time.Since(start))
} else if res.RowsAffected > 1 {
log.Infof("%s: updated %d previews [%s]", name, res.RowsAffected, time.Since(start))
} else if res.RowsAffected > 0 {
log.Infof("%s: %s updated [%s]", name, english.Plural(int(res.RowsAffected), "preview", "previews"), time.Since(start))
}
}

4
internal/entity/photo_counts.go
View file

@ -148,7 +148,7 @@ func UpdateLabelPhotoCounts() (err error) {
func UpdatePhotoCounts() (err error) {
if err = UpdatePlacesPhotoCounts(); err != nil {
if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("counts: failed updating places, incompatible database version")
log.Errorf("counts: failed updating places, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -158,7 +158,7 @@ func UpdatePhotoCounts() (err error) {
if err = UpdateSubjectFileCounts(); err != nil {
if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("counts: failed updating subjects, incompatible database version")
log.Errorf("counts: failed updating subjects, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}

6
internal/face/detector_test.go
View file

@ -5,6 +5,8 @@ import (
"path/filepath"
"testing"
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/pkg/fastwalk"
"github.com/stretchr/testify/assert"
)
@ -46,7 +48,7 @@ func TestDetect(t *testing.T) {
t.Fatal(err)
}
t.Logf("found %d faces in '%s'", len(faces), baseName)
t.Logf("found %s in '%s'", english.Plural(len(faces), "face", "faces"), baseName)
if len(faces) > 0 {
// t.Logf("results: %#v", faces)
@ -99,7 +101,7 @@ func TestDetectOverlap(t *testing.T) {
t.Fatal(err)
}
t.Logf("found %d faces in '%s'", len(faces), baseName)
t.Logf("found %s in '%s'", english.Plural(len(faces), "face", "faces"), baseName)
if len(faces) > 0 {
// t.Logf("results: %#v", faces)

8
internal/photoprism/cleanup.go
View file

@ -8,6 +8,8 @@ import (
"runtime/debug"
"strings"
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/config"
@ -136,7 +138,7 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
if files, err := query.OrphanFiles(); err != nil {
log.Errorf("cleanup: %s (find orphan files)", err)
} else if l := len(files); l > 0 {
log.Infof("cleanup: found %d orphan files", l)
log.Infof("cleanup: found %s", english.Plural(l, "orphan file", "orphan files"))
} else {
log.Infof("cleanup: found no orphan files")
}
@ -148,14 +150,14 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
// Only update counts if anything was deleted.
if len(deleted) > 0 {
log.Info("cleanup: updating photo counts")
log.Info("updating photo counts")
// Update precalculated photo and file counts.
if err := entity.UpdatePhotoCounts(); err != nil {
log.Errorf("cleanup: %s (update counts)", err)
}
log.Info("cleanup: updating preview images")
log.Info("updating preview thumbs")
// Update album, subject, and label preview thumbs.
if err := query.UpdatePreviews(); err != nil {

13
internal/photoprism/faces_audit.go
View file

@ -1,6 +1,7 @@
package photoprism
import (
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/query"
@ -123,11 +124,11 @@ func (w *Faces) Audit(fix bool) (err error) {
} else if l := len(orphans); l == 0 {
log.Infof("found no orphan face clusters")
} else if !fix {
log.Infof("found %d orphan face clusters", l)
log.Infof("found %s", english.Plural(l, "orphan face cluster", "orphan face clusters"))
} else if err := orphans.Delete(); err != nil {
log.Errorf("failed removing %d orphan face clusters: %s", l, err)
log.Errorf("failed removing %s: %s", english.Plural(l, "orphan face cluster", "orphan face clusters"), err)
} else {
log.Infof("removed %d orphan face clusters", l)
log.Infof("removed %s", english.Plural(l, "orphan face cluster", "orphan face clusters"))
}
// Find and fix orphan people.
@ -136,11 +137,11 @@ func (w *Faces) Audit(fix bool) (err error) {
} else if l := len(orphans); l == 0 {
log.Infof("found no orphan people")
} else if !fix {
log.Infof("found %d orphan people", l)
log.Infof("found %s", english.Plural(l, "orphan person", "orphan people"))
} else if err := orphans.Delete(); err != nil {
log.Errorf("failed fixing %d orphan people: %s", l, err)
log.Errorf("failed fixing %s: %s", english.Plural(l, "orphan person", "orphan people"), err)
} else {
log.Infof("removed %d orphan people", l)
log.Infof("removed %s", english.Plural(l, "orphan person", "orphan people"))
}
return nil

6
internal/photoprism/faces_cluster.go
View file

@ -3,6 +3,8 @@ package photoprism
import (
"fmt"
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/query"
@ -46,8 +48,8 @@ func (w *Faces) Cluster(opt FacesOptions) (added entity.Faces, err error) {
sizes := c.Sizes()
if len(sizes) > 1 {
log.Infof("faces: found %d new clusters", len(sizes))
if len(sizes) > 0 {
log.Infof("faces: found %s", english.Plural(len(sizes), "new cluster", "new clusters"))
} else {
log.Debugf("faces: found no new clusters")
}

8
internal/photoprism/index_faces.go
View file

@ -3,6 +3,8 @@ package photoprism
import (
"time"
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/thumb"
@ -44,10 +46,8 @@ func (ind *Index) Faces(jpeg *MediaFile, expected int) face.Faces {
log.Debugf("%s in %s", err, txt.Quote(jpeg.BaseName()))
}
if l := len(faces); l == 1 {
log.Infof("index: found %d face in %s [%s]", l, txt.Quote(jpeg.BaseName()), time.Since(start))
} else if l > 1 {
log.Infof("index: found %d faces in %s [%s]", l, txt.Quote(jpeg.BaseName()), time.Since(start))
if l := len(faces); l > 0 {
log.Infof("index: found %s in %s [%s]", english.Plural(l, "face", "faces"), txt.Quote(jpeg.BaseName()), time.Since(start))
}
return faces

12
internal/photoprism/purge.go
View file

@ -6,6 +6,8 @@ import (
"runtime/debug"
"time"
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
@ -239,13 +241,13 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
time.Sleep(50 * time.Millisecond)
}
log.Info("purge: searching index for unassigned primary files")
log.Info("searching index for unassigned primary files")
if err := query.FixPrimaries(); err != nil {
log.Errorf("purge: %s (fix primary files)", err.Error())
}
log.Info("purge: searching index for hidden media files")
log.Info("searching index for hidden media files")
// Set photo quality scores to -1 if files are missing.
if err := query.ResetPhotoQuality(); err != nil {
@ -257,7 +259,7 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
if files, err := query.OrphanFiles(); err != nil {
log.Errorf("purge: %s (find orphan files)", err)
} else if l := len(files); l > 0 {
log.Infof("purge: found %d orphan files", l)
log.Infof("purge: found %s", english.Plural(l, "orphan file", "orphan files"))
} else {
log.Infof("purge: found no orphan files")
}
@ -273,13 +275,13 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
}
// Update precalculated photo and file counts.
log.Info("purge: updating photo counts")
log.Info("updating photo counts")
if err := entity.UpdatePhotoCounts(); err != nil {
log.Errorf("purge: %s (update counts)", err)
}
// Update album, subject, and label preview thumbs.
log.Info("purge: updating preview images")
log.Info("updating preview thumbs")
if err := query.UpdatePreviews(); err != nil {
log.Errorf("purge: %s (update previews)", err)
}

260
internal/query/previews.go
View file

@ -5,7 +5,9 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize/english"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
)
@ -13,19 +15,38 @@ import (
func UpdateAlbumDefaultPreviews() (err error) {
start := time.Now()
err = Db().Table(entity.Album{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
var res *gorm.DB
switch DbDialect() {
case MySQL:
res = Db().Exec(`UPDATE albums LEFT JOIN (
SELECT p2.album_uid, f.file_hash FROM files f, (
SELECT pa.album_uid, max(p.id) AS photo_id FROM photos p
JOIN photos_albums pa ON pa.photo_uid = p.photo_uid AND pa.hidden = 0
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY pa.album_uid) p2 WHERE p2.photo_id = f.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
) b ON b.album_uid = albums.album_uid
SET thumb = b.file_hash WHERE album_type = ? AND (thumb_src = ? OR thumb IS NULL OR thumb = '')
AND thumb <> b.file_hash`, entity.AlbumDefault, entity.SrcAuto)
case SQLite:
res = Db().Table(entity.Album{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f
JOIN photos_albums pa ON pa.album_uid = albums.album_uid AND pa.photo_uid = f.photo_uid AND pa.hidden = 0
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1
WHERE f.deleted_at IS NULL AND f.file_missing = 0 AND f.file_hash <> '' AND f.file_primary = 1 AND f.file_type = 'jpg'
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality >= 0
WHERE f.deleted_at IS NULL AND f.file_missing = 0 AND f.file_hash <> '' AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.taken_at DESC LIMIT 1
) WHERE thumb_src='' AND album_type = 'album' AND deleted_at IS NULL`)).Error
) WHERE album_type = ? AND (thumb_src = ? OR thumb = '' OR thumb IS NULL)`, entity.AlbumDefault, entity.SrcDefault))
default:
return nil
}
err = res.Error
if err == nil {
log.Debugf("previews: updated albums [%s]", time.Since(start))
log.Debugf("previews: %s updated [%s]", english.Plural(int(res.RowsAffected), "album", "albums"), time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating albums, incompatible database version")
log.Errorf("previews: failed updating albums, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -37,19 +58,38 @@ func UpdateAlbumDefaultPreviews() (err error) {
func UpdateAlbumFolderPreviews() (err error) {
start := time.Now()
err = Db().Table(entity.Album{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f
JOIN photos p ON p.id = f.photo_id AND p.photo_path = albums.album_path AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1
WHERE f.deleted_at IS NULL AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.taken_at DESC LIMIT 1
) WHERE thumb_src = '' AND album_type = 'folder' AND deleted_at IS NULL`)).
Error
var res *gorm.DB
switch DbDialect() {
case MySQL:
res = Db().Exec(`UPDATE albums LEFT JOIN (
SELECT p2.photo_path, f.file_hash FROM files f, (
SELECT p.photo_path, max(p.id) AS photo_id FROM photos p
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY p.photo_path) p2 WHERE p2.photo_id = f.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
) b ON b.photo_path = albums.album_path
SET thumb = b.file_hash WHERE album_type = ? AND (thumb_src = ? OR thumb IS NULL OR thumb = '')
AND thumb <> b.file_hash`, entity.AlbumFolder, entity.SrcAuto)
case SQLite:
res = Db().Table(entity.Album{}.TableName()).UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f,(
SELECT p.photo_path, max(p.id) AS photo_id FROM photos p
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY p.photo_path
) b
WHERE f.photo_id = b.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
AND b.photo_path = albums.album_path LIMIT 1)
WHERE album_type = ? AND (thumb_src = ? OR thumb IS NULL OR thumb = '')`, entity.AlbumFolder, entity.SrcAuto))
default:
return nil
}
err = res.Error
if err == nil {
log.Debugf("previews: updated folders [%s]", time.Since(start))
log.Debugf("previews: %s updated [%s]", english.Plural(int(res.RowsAffected), "folder", "folders"), time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating folders, incompatible database version")
log.Errorf("previews: failed updating folders, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -61,43 +101,38 @@ func UpdateAlbumFolderPreviews() (err error) {
func UpdateAlbumMonthPreviews() (err error) {
start := time.Now()
err = Db().Table(entity.Album{}.TableName()).
Where("album_type = ?", entity.AlbumMonth).
Where("thumb IS NOT NULL AND thumb_src = ?", entity.SrcAuto).
UpdateColumns(entity.Values{"thumb": nil}).Error
/* TODO: Slow with many photos due to missing index.
var res *gorm.DB
switch DbDialect() {
case MySQL:
err = Db().Table(entity.Album{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f JOIN photos p ON p.id = f.photo_id
WHERE YEAR(p.taken_at) = albums.album_year AND MONTH(p.taken_at) = albums.album_month
AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1 AND f.deleted_at IS NULL
AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.taken_at DESC LIMIT 1
) WHERE thumb IS NULL AND thumb_src = '' AND album_type = 'month' AND deleted_at IS NULL`)).
Error
res = Db().Exec(`UPDATE albums LEFT JOIN (
SELECT p2.photo_year, p2.photo_month, f.file_hash FROM files f, (
SELECT p.photo_year, p.photo_month, max(p.id) AS photo_id FROM photos p
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY p.photo_year, p.photo_month) p2 WHERE p2.photo_id = f.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
) b ON b.photo_year = albums.album_year AND b.photo_month = albums.album_month
SET thumb = b.file_hash WHERE album_type = ?
AND thumb <> b.file_hash AND (thumb_src = ? OR thumb IS NULL OR thumb = '')`, entity.AlbumMonth, entity.SrcAuto)
case SQLite:
err = Db().Table(entity.Album{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f JOIN photos p ON p.id = f.photo_id
WHERE strftime('%Y%m', p.taken_at) = (albums.album_year || printf('%02d', albums.album_month))
AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1 AND f.deleted_at IS NULL
AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.taken_at DESC LIMIT 1
) WHERE thumb IS NULL AND thumb_src = '' AND album_type = 'month' AND deleted_at IS NULL`)).
Error
res = Db().Table(entity.Album{}.TableName()).UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f,(
SELECT p.photo_year, p.photo_month, max(p.id) AS photo_id FROM photos p
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY p.photo_year, p.photo_month
) b
WHERE f.photo_id = b.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
AND b.photo_year = albums.album_year AND b.photo_month = albums.album_month LIMIT 1)
WHERE album_type = ? AND (thumb_src = ? OR thumb = '' OR thumb IS NULL)`, entity.AlbumMonth, entity.SrcAuto))
default:
return nil
}
*/
err = res.Error
if err == nil {
log.Debugf("previews: updated calendar [%s]", time.Since(start))
log.Debugf("previews: %s updated [%s]", english.Plural(int(res.RowsAffected), "month", "months"), time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating calendar, incompatible database version")
log.Errorf("previews: failed updating calendar, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -129,48 +164,57 @@ func UpdateAlbumPreviews() (err error) {
func UpdateLabelPreviews() (err error) {
start := time.Now()
// Labels.
err = Db().Table(entity.Label{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f
var res *gorm.DB
switch DbDialect() {
case MySQL:
res = Db().Exec(`UPDATE labels JOIN (
SELECT p2.label_id, f.file_hash FROM files f, (
SELECT pl.label_id as label_id, max(p.id) AS photo_id FROM photos p
JOIN photos_labels pl ON pl.photo_id = p.id AND pl.uncertainty < 100
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY pl.label_id
UNION
SELECT c.category_id as label_id, max(p.id) AS photo_id FROM photos p
JOIN photos_labels pl ON pl.photo_id = p.id AND pl.uncertainty < 100
JOIN categories c ON c.label_id = pl.label_id
WHERE p.photo_quality >= 0 AND p.photo_private = 0 AND p.deleted_at IS NULL
GROUP BY c.category_id
) p2 WHERE p2.photo_id = f.photo_id AND f.file_primary = 1 AND f.file_type = 'jpg'
) b ON b.label_id = labels.id
SET thumb = b.file_hash WHERE thumb_src = '' OR thumb IS NULL OR thumb = ''`)
case SQLite:
res = Db().Table(entity.Label{}.TableName()).UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f
JOIN photos_labels pl ON pl.label_id = labels.id AND pl.photo_id = f.photo_id AND pl.uncertainty < 100
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality >= 0
WHERE f.deleted_at IS NULL AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.photo_quality DESC, pl.uncertainty ASC, p.taken_at DESC LIMIT 1
) WHERE thumb_src = '' AND deleted_at IS NULL`)).
Error
) WHERE (thumb_src = ? OR thumb = '' OR thumb IS NULL)`, entity.SrcAuto))
if err == nil {
log.Debugf("previews: updated labels [%s]", time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating labels, incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
return err
}
// UpdateCategoryPreviews updates category preview images.
func UpdateCategoryPreviews() (err error) {
start := time.Now()
// Categories.
err = Db().Table(entity.Label{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
if res.Error == nil {
catRes := Db().Table(entity.Label{}.TableName()).
UpdateColumn("thumb", gorm.Expr(`(
SELECT f.file_hash FROM files f
JOIN photos_labels pl ON pl.photo_id = f.photo_id AND pl.uncertainty < 100
JOIN categories c ON c.label_id = pl.label_id AND c.category_id = labels.id
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality > -1
JOIN photos p ON p.id = f.photo_id AND p.photo_private = 0 AND p.deleted_at IS NULL AND p.photo_quality >= 0
WHERE f.deleted_at IS NULL AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg'
ORDER BY p.photo_quality DESC, pl.uncertainty ASC, p.taken_at DESC LIMIT 1
) WHERE thumb IS NULL AND thumb_src = '' AND deleted_at IS NULL`)).
Error
) WHERE thumb IS NULL`))
res.RowsAffected += catRes.RowsAffected
}
default:
return nil
}
err = res.Error
if err == nil {
log.Debugf("previews: updated categories [%s]", time.Since(start))
log.Debugf("previews: %s updated [%s]", english.Plural(int(res.RowsAffected), "label", "labels"), time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating categories, incompatible database version")
log.Errorf("previews: failed updating labels, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -182,47 +226,38 @@ func UpdateCategoryPreviews() (err error) {
func UpdateSubjectPreviews() (err error) {
start := time.Now()
/* Previous implementation for reference:
var res *gorm.DB
return Db().Table(entity.Subject{}.TableName()).
UpdateColumn("thumb", gorm.Expr("(SELECT f.file_hash FROM files f "+
fmt.Sprintf(
"JOIN %s m ON f.file_uid = m.file_uid AND m.subj_uid = %s.subj_uid",
entity.Marker{}.TableName(),
entity.Subject{}.TableName())+
` JOIN photos p ON f.photo_id = p.id
WHERE m.marker_invalid = 0 AND f.deleted_at IS NULL AND f.file_hash <> '' AND p.deleted_at IS NULL
AND f.file_primary = 1 AND f.file_missing = 0 AND p.photo_private = 0 AND p.photo_quality > -1
ORDER BY p.taken_at DESC LIMIT 1)
WHERE thumb_src='' AND deleted_at IS NULL`)).
Error */
// TODO: Avoid using private photos as subject covers.
switch DbDialect() {
case MySQL:
res = Db().Exec(`UPDATE ? LEFT JOIN (
SELECT m.subj_uid, m.q, MAX(m.thumb) AS marker_thumb FROM ? m
WHERE m.subj_uid <> '' AND m.subj_uid IS NOT NULL
AND m.marker_invalid = 0 AND m.thumb IS NOT NULL AND m.thumb <> ''
GROUP BY m.subj_uid, m.q
) b ON b.subj_uid = subjects.subj_uid
SET thumb = marker_thumb WHERE subjects.subj_type = ? AND (thumb_src = ? OR thumb IS NULL OR thumb = '')`,
gorm.Expr(entity.Subject{}.TableName()), gorm.Expr(entity.Marker{}.TableName()), entity.SubjPerson, entity.SrcAuto)
case SQLite:
res = Db().Table(entity.Subject{}.TableName()).UpdateColumn("thumb", gorm.Expr(
"(SELECT m.thumb FROM "+
fmt.Sprintf(
"%s m WHERE m.subj_uid = %s.subj_uid ",
entity.Marker{}.TableName(),
entity.Subject{}.TableName())+
` AND m.thumb <> '' ORDER BY m.subj_src DESC, m.q DESC LIMIT 1)
WHERE subjects.subj_type = ? AND (thumb_src = ? OR thumb = '' OR thumb IS NULL)`, entity.SubjPerson, entity.SrcAuto))
default:
return nil
}
err = Db().Table(entity.Subject{}.TableName()).
UpdateColumn("thumb", gorm.Expr("(SELECT m.thumb FROM "+
fmt.Sprintf(
"%s m WHERE m.subj_uid = %s.subj_uid ",
entity.Marker{}.TableName(),
entity.Subject{}.TableName())+
` AND m.thumb <> '' ORDER BY m.subj_src DESC, m.q DESC LIMIT 1)
WHERE (thumb_src = '' OR thumb_src IS NULL) AND deleted_at IS NULL`)).
Error
/** err = Db().Table(entity.Subject{}.TableName()).
UpdateColumn("thumb", gorm.Expr("(SELECT m.file_hash FROM "+
fmt.Sprintf(
"%s m WHERE m.subj_uid = %s.subj_uid AND m.subj_src = 'manual' ",
entity.Marker{}.TableName(),
entity.Subject{}.TableName())+
` AND m.file_hash <> '' ORDER BY m.w DESC LIMIT 1)
WHERE thumb_src = '' AND deleted_at IS NULL`)).
Error
*/
err = res.Error
if err == nil {
log.Debugf("previews: updated subjects [%s]", time.Since(start))
log.Debugf("previews: %s updated [%s]", english.Plural(int(res.RowsAffected), "subject", "subjects"), time.Since(start))
} else if strings.Contains(err.Error(), "Error 1054") {
log.Errorf("previews: failed updating subjects, incompatible database version")
log.Errorf("previews: failed updating subjects, potentially incompatible database version")
log.Errorf("%s see https://jira.mariadb.org/browse/MDEV-25362", err)
return nil
}
@ -242,11 +277,6 @@ func UpdatePreviews() (err error) {
return err
}
// Update Categories.
if err = UpdateCategoryPreviews(); err != nil {
return err
}
// Update Subjects.
if err = UpdateSubjectPreviews(); err != nil {
return err

6
internal/search/photos.go
View file

@ -5,6 +5,8 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize/english"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
@ -96,7 +98,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
return results, 0, result.Error
}
log.Infof("photos: found %d results for %s [%s]", len(results), f.SerializeAll(), time.Since(start))
log.Infof("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start))
if f.Merged {
return results.Merged()
@ -476,7 +478,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
return results, 0, err
}
log.Infof("photos: found %d results for %s [%s]", len(results), f.SerializeAll(), time.Since(start))
log.Infof("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start))
if f.Merged {
return results.Merged()

4
internal/search/photos_geo.go
View file

@ -5,6 +5,8 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize/english"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
@ -293,7 +295,7 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) {
return results, result.Error
}
log.Infof("geo: found %d photos for %s [%s]", len(results), f.SerializeAll(), time.Since(start))
log.Infof("geo: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start))
return results, nil
}

4
internal/workers/meta.go
View file

@ -127,14 +127,14 @@ func (m *Meta) Start(delay time.Duration) (err error) {
log.Warn(err)
}
log.Debugf("metadata: updating photo counts")
log.Debugf("updating photo counts")
// Update precalculated photo and file counts.
if err := entity.UpdatePhotoCounts(); err != nil {
log.Warnf("metadata: %s (update counts)", err.Error())
}
log.Debugf("metadata: updating preview images")
log.Debugf("updating preview thumbs")
// Update album, subject, and label preview thumbs.
if err := query.UpdatePreviews(); err != nil {