From 252e67ce03a5fd0ddf209c73c4866f0f36e8bf01 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 9 May 2020 11:00:22 +0200 Subject: [PATCH] Add photo_count column to labels table Signed-off-by: Michael Mayer --- Dockerfile | 2 +- Makefile | 4 ++- docker-compose.travis.yml | 2 +- docker-compose.yml | 2 +- docker/development/.my.cnf | 5 ++++ docker/development/Dockerfile | 3 ++ docker/photoprism/Dockerfile | 4 +-- docker/photoprism/arm64/Dockerfile | 2 +- internal/api/batch.go | 20 +++++++++++-- internal/api/photo_label.go | 8 ++++++ internal/entity/entity.go | 10 +++---- internal/entity/label.go | 1 + internal/query/counts.go | 18 ------------ internal/query/labels.go | 6 ++-- internal/query/photo_counts.go | 28 +++++++++++++++++++ .../{counts_test.go => photo_counts_test.go} | 0 scripts/{test-db.sql => reset-test-db.sql} | 23 ++++++++++----- 17 files changed, 95 insertions(+), 43 deletions(-) create mode 100644 docker/development/.my.cnf delete mode 100644 internal/query/counts.go create mode 100644 internal/query/photo_counts.go rename internal/query/{counts_test.go => photo_counts_test.go} (100%) rename scripts/{test-db.sql => reset-test-db.sql} (54%) diff --git a/Dockerfile b/Dockerfile index dd9b021ec..f20f3ddb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM photoprism/development:20200427 +FROM photoprism/development:20200509 # Set up project directory WORKDIR "/go/src/github.com/photoprism/photoprism" diff --git a/Makefile b/Makefile index ee1bc8e57..7d161253e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ all: dep build dep: dep-tensorflow dep-js dep-go build: generate build-js build-go install: install-bin install-assets -test: test-js test-go +test: reset-test-db test-js test-go acceptance-all: start acceptance acceptance-firefox stop test-all: test acceptance-all fmt: fmt-js fmt-go @@ -90,6 +90,8 @@ acceptance: acceptance-firefox: $(info Running JS acceptance tests in Firefox...) (cd frontend && npm run acceptance-firefox) +reset-test-db: + mysql < scripts/reset-test-db.sql test-go: $(info Running all Go unit tests...) $(GOTEST) -parallel 1 -count 1 -cpu 1 -tags slow -timeout 20m ./pkg/... ./internal/... diff --git a/docker-compose.travis.yml b/docker-compose.travis.yml index 33102794c..b9e4e6ccc 100644 --- a/docker-compose.travis.yml +++ b/docker-compose.travis.yml @@ -48,7 +48,7 @@ services: expose: - "4001" volumes: - - "./scripts/test-db.sql:/docker-entrypoint-initdb.d/test-db.sql" + - "./scripts/reset-test-db.sql:/docker-entrypoint-initdb.d/reset-test-db.sql" environment: MYSQL_ROOT_PASSWORD: photoprism MYSQL_USER: photoprism diff --git a/docker-compose.yml b/docker-compose.yml index b9e08ec9c..85ce9da69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,7 @@ services: ports: - "4001:4001" # MySQL (for tests) volumes: - - "./scripts/test-db.sql:/docker-entrypoint-initdb.d/test-db.sql" + - "./scripts/reset-test-db.sql:/docker-entrypoint-initdb.d/reset-test-db.sql" environment: MYSQL_ROOT_PASSWORD: photoprism MYSQL_USER: photoprism diff --git a/docker/development/.my.cnf b/docker/development/.my.cnf new file mode 100644 index 000000000..37635bf06 --- /dev/null +++ b/docker/development/.my.cnf @@ -0,0 +1,5 @@ +[client] +user=root +password=photoprism +host=photoprism-db +port=4001 diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 43f304500..e792c618e 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -116,6 +116,9 @@ RUN env GO111MODULE=off /usr/local/go/bin/go get -u github.com/tsliwowicz/go-wrk RUN env GO111MODULE=off /usr/local/go/bin/go get -u github.com/kyoh86/richgo RUN echo "alias go=richgo" > /root/.bash_aliases +# MariaDB test database settings +COPY /docker/development/.my.cnf /root/.my.cnf + # Set up project directory WORKDIR "/go/src/github.com/photoprism/photoprism" diff --git a/docker/photoprism/Dockerfile b/docker/photoprism/Dockerfile index e2a2a4bf5..36d844e28 100644 --- a/docker/photoprism/Dockerfile +++ b/docker/photoprism/Dockerfile @@ -1,4 +1,4 @@ -FROM photoprism/development:20200427 as build +FROM photoprism/development:20200509 as build # Set up project directory WORKDIR "/go/src/github.com/photoprism/photoprism" @@ -85,7 +85,7 @@ RUN chmod -R 777 /photoprism RUN photoprism -v # Expose http and database ports -EXPOSE 2342 2343 +EXPOSE 2342 2343 4000 # Run server CMD photoprism start diff --git a/docker/photoprism/arm64/Dockerfile b/docker/photoprism/arm64/Dockerfile index 42268ce3e..66ebf5bf1 100644 --- a/docker/photoprism/arm64/Dockerfile +++ b/docker/photoprism/arm64/Dockerfile @@ -169,7 +169,7 @@ RUN chmod -R 777 /photoprism RUN photoprism -v # Expose http and database ports -EXPOSE 2342 2343 +EXPOSE 2342 2343 4000 # Run server CMD photoprism start diff --git a/internal/api/batch.go b/internal/api/batch.go index 223919901..f078c0995 100644 --- a/internal/api/batch.go +++ b/internal/api/batch.go @@ -41,7 +41,12 @@ func BatchPhotosArchive(router *gin.RouterGroup, conf *config.Config) { log.Infof("photos: archiving %#v", f.Photos) - entity.Db().Where("photo_uuid IN (?)", f.Photos).Delete(&entity.Photo{}) + err := entity.Db().Where("photo_uuid IN (?)", f.Photos).Delete(&entity.Photo{}).Error + + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, ErrSaveFailed) + return + } if err := query.UpdatePhotoCounts(); err != nil { log.Errorf("photos: %s", err) @@ -82,8 +87,13 @@ func BatchPhotosRestore(router *gin.RouterGroup, conf *config.Config) { log.Infof("restoring photos: %#v", f.Photos) - entity.Db().Unscoped().Model(&entity.Photo{}).Where("photo_uuid IN (?)", f.Photos). - UpdateColumn("deleted_at", gorm.Expr("NULL")) + err := entity.Db().Unscoped().Model(&entity.Photo{}).Where("photo_uuid IN (?)", f.Photos). + UpdateColumn("deleted_at", gorm.Expr("NULL")).Error + + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, ErrSaveFailed) + return + } if err := query.UpdatePhotoCounts(); err != nil { log.Errorf("photos: %s", err) @@ -165,6 +175,10 @@ func BatchPhotosPrivate(router *gin.RouterGroup, conf *config.Config) { return } + if err := query.UpdatePhotoCounts(); err != nil { + log.Errorf("photos: %s", err) + } + if entities, err := query.PhotoSelection(f); err == nil { event.EntitiesUpdated("photos", entities) } diff --git a/internal/api/photo_label.go b/internal/api/photo_label.go index 00e86d364..d52ec47d3 100644 --- a/internal/api/photo_label.go +++ b/internal/api/photo_label.go @@ -71,6 +71,10 @@ func AddPhotoLabel(router *gin.RouterGroup, conf *config.Config) { return } + if err := query.UpdatePhotoCounts(); err != nil { + log.Errorf("photo: %s", err) + } + PublishPhotoEvent(EntityUpdated, c.Param("uuid"), c) event.Success("label updated") @@ -131,6 +135,10 @@ func RemovePhotoLabel(router *gin.RouterGroup, conf *config.Config) { return } + if err := query.UpdatePhotoCounts(); err != nil { + log.Errorf("photo: %s", err) + } + PublishPhotoEvent(EntityUpdated, c.Param("uuid"), c) event.Success("label removed") diff --git a/internal/entity/entity.go b/internal/entity/entity.go index 049141c33..0c613004e 100644 --- a/internal/entity/entity.go +++ b/internal/entity/entity.go @@ -57,10 +57,10 @@ func (list Types) WaitForMigration() { for name := range list { for i := 0; i <= attempts; i++ { if err := Db().Raw(fmt.Sprintf("DESCRIBE `%s`", name)).Scan(&struct{}{}).Error; err == nil { - log.Debugf("entity: table %s migrated", name) + // log.Debugf("entity: table %s migrated", name) break } else { - log.Debugf("entity: %s", err.Error()) + log.Debugf("entity: wait for migration %s (%s)", err.Error(), name) } if i == attempts { @@ -78,8 +78,8 @@ func (list Types) Truncate() { if err := Db().Raw(fmt.Sprintf("TRUNCATE TABLE `%s`", name)).Scan(&struct{}{}).Error; err == nil { log.Debugf("entity: removed all data from %s", name) break - } else { - log.Debugf("entity: %s", err.Error()) + } else if err.Error() != "record not found" { + log.Debugf("entity: truncate %s (%s)", err.Error(), name) } } } @@ -88,7 +88,7 @@ func (list Types) Truncate() { func (list Types) Migrate() { for _, entity := range list { if err := UnscopedDb().AutoMigrate(entity).Error; err != nil { - log.Debugf("entity: %s (waiting 1s)", err.Error()) + log.Debugf("entity: migrate %s (waiting 1s)", err.Error()) time.Sleep(time.Second) diff --git a/internal/entity/label.go b/internal/entity/label.go index 2ecdb2068..284374a97 100644 --- a/internal/entity/label.go +++ b/internal/entity/label.go @@ -23,6 +23,7 @@ type Label struct { LabelNotes string `gorm:"type:text;"` LabelCategories []*Label `gorm:"many2many:categories;association_jointable_foreignkey:category_id"` Links []Link `gorm:"foreignkey:ShareUUID;association_foreignkey:LabelUUID"` + PhotoCount int CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time `sql:"index"` diff --git a/internal/query/counts.go b/internal/query/counts.go deleted file mode 100644 index d497aeecb..000000000 --- a/internal/query/counts.go +++ /dev/null @@ -1,18 +0,0 @@ -package query - -import "github.com/jinzhu/gorm" - -// UpdatePhotoCounts updates photos count in related tables as needed. -func UpdatePhotoCounts() error { - /* - UPDATE places - SET - photo_count = (SELECT - COUNT(*) FROM - photos ph - WHERE places.id = ph.place_id AND ph.photo_quality >= 0 AND ph.deleted_at IS NULL) - */ - - return Db().Table("places"). - UpdateColumn("photo_count", gorm.Expr("(SELECT COUNT(*) FROM photos ph WHERE places.id = ph.place_id AND ph.photo_quality >= 0 AND ph.deleted_at IS NULL)")).Error -} diff --git a/internal/query/labels.go b/internal/query/labels.go index e517137b3..6f1199d7d 100644 --- a/internal/query/labels.go +++ b/internal/query/labels.go @@ -43,7 +43,7 @@ func LabelByUUID(labelUUID string) (label entity.Label, err error) { func LabelThumbBySlug(labelSlug string) (file entity.File, err error) { if err := Db().Where("files.file_primary AND files.deleted_at IS NULL"). Joins("JOIN labels ON labels.label_slug = ?", labelSlug). - Joins("JOIN photos_labels ON photos_labels.label_id = labels.id AND photos_labels.photo_id = files.photo_id"). + Joins("JOIN photos_labels ON photos_labels.label_id = labels.id AND photos_labels.photo_id = files.photo_id AND photos_labels.uncertainty < 100"). Joins("JOIN photos ON photos.id = files.photo_id AND photos.photo_private = 0 AND photos.deleted_at IS NULL"). Order("photos.photo_quality DESC, photos_labels.uncertainty ASC"). First(&file).Error; err != nil { @@ -58,7 +58,7 @@ func LabelThumbByUUID(labelUUID string) (file entity.File, err error) { // Search matching label err = Db().Where("files.file_primary AND files.deleted_at IS NULL"). Joins("JOIN labels ON labels.label_uuid = ?", labelUUID). - Joins("JOIN photos_labels ON photos_labels.label_id = labels.id AND photos_labels.photo_id = files.photo_id"). + Joins("JOIN photos_labels ON photos_labels.label_id = labels.id AND photos_labels.photo_id = files.photo_id AND photos_labels.uncertainty < 100"). Joins("JOIN photos ON photos.id = files.photo_id AND photos.photo_private = 0 AND photos.deleted_at IS NULL"). Order("photos.photo_quality DESC, photos_labels.uncertainty ASC"). First(&file).Error @@ -69,7 +69,7 @@ func LabelThumbByUUID(labelUUID string) (file entity.File, err error) { // If failed, search for category instead err = Db().Where("files.file_primary AND files.deleted_at IS NULL"). - Joins("JOIN photos_labels ON photos_labels.photo_id = files.photo_id"). + Joins("JOIN photos_labels ON photos_labels.photo_id = files.photo_id AND photos_labels.uncertainty < 100"). Joins("JOIN categories c ON photos_labels.label_id = c.label_id"). Joins("JOIN labels ON c.category_id = labels.id AND labels.label_uuid= ?", labelUUID). Joins("JOIN photos ON photos.id = files.photo_id AND photos.photo_private = 0 AND photos.deleted_at IS NULL"). diff --git a/internal/query/photo_counts.go b/internal/query/photo_counts.go new file mode 100644 index 000000000..2d98dcc1c --- /dev/null +++ b/internal/query/photo_counts.go @@ -0,0 +1,28 @@ +package query + +import "github.com/jinzhu/gorm" + +// UpdatePhotoCounts updates photos count in related tables as needed. +func UpdatePhotoCounts() error { + if err := Db().Table("places"). + UpdateColumn("photo_count", gorm.Expr("(SELECT COUNT(*) FROM photos ph " + + "WHERE places.id = ph.place_id " + + "AND ph.photo_quality >= 0 " + + "AND ph.photo_private = 0 " + + "AND ph.deleted_at IS NULL)")).Error; err != nil { + return err + } + + if err := Db().Table("labels"). + UpdateColumn("photo_count", gorm.Expr("(SELECT COUNT(*) FROM photos_labels " + + "JOIN photos ph ON photos_labels.photo_id = ph.id " + + "WHERE photos_labels.label_id = labels.id " + + "AND photos_labels.uncertainty < 100 " + + "AND ph.photo_quality >= 0 " + + "AND ph.photo_private = 0 " + + "AND ph.deleted_at IS NULL)")).Error; err != nil { + return err + } + + return nil +} diff --git a/internal/query/counts_test.go b/internal/query/photo_counts_test.go similarity index 100% rename from internal/query/counts_test.go rename to internal/query/photo_counts_test.go diff --git a/scripts/test-db.sql b/scripts/reset-test-db.sql similarity index 54% rename from scripts/test-db.sql rename to scripts/reset-test-db.sql index 4744f2f70..ee988227f 100644 --- a/scripts/test-db.sql +++ b/scripts/reset-test-db.sql @@ -1,9 +1,18 @@ -CREATE DATABASE IF NOT EXISTS api; -CREATE DATABASE IF NOT EXISTS config; -CREATE DATABASE IF NOT EXISTS entity; +DROP DATABASE IF EXISTS photoprism; CREATE DATABASE IF NOT EXISTS photoprism; -CREATE DATABASE IF NOT EXISTS query; -CREATE DATABASE IF NOT EXISTS remote; -CREATE DATABASE IF NOT EXISTS service; -CREATE DATABASE IF NOT EXISTS workers; +DROP DATABASE IF EXISTS acceptance; CREATE DATABASE IF NOT EXISTS acceptance; +DROP DATABASE IF EXISTS api; +CREATE DATABASE IF NOT EXISTS api; +DROP DATABASE IF EXISTS config; +CREATE DATABASE IF NOT EXISTS config; +DROP DATABASE IF EXISTS entity; +CREATE DATABASE IF NOT EXISTS entity; +DROP DATABASE IF EXISTS query; +CREATE DATABASE IF NOT EXISTS query; +DROP DATABASE IF EXISTS remote; +CREATE DATABASE IF NOT EXISTS remote; +DROP DATABASE IF EXISTS service; +CREATE DATABASE IF NOT EXISTS service; +DROP DATABASE IF EXISTS workers; +CREATE DATABASE IF NOT EXISTS workers;