beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Backend: Use original file if thumb size exceeds limit #172

Browse source 
Plus some mutex and config refactoring along the way...

Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-01-08 19:51:21 +01:00
parent e734fd9049
commit b37d4472e4
26 changed files with 203 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
internal/api/album.go
View file

@ -443,11 +443,22 @@ func AlbumThumbnail(router *gin.RouterGroup, conf *config.Config) {
return
}
// Use original file if thumb size exceeds limit, see https://github.com/photoprism/photoprism/issues/157
if thumbType.Height > thumb.MaxHeight || thumbType.Width > thumb.MaxWidth {
log.Debugf("album: using original, thumbnail size exceeds limit (width %d, height %d)", thumbType.Width, thumbType.Height)
if c.Query("download") != "" {
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", f.DownloadFileName()))
}
c.File(fileName)
return
}
if thumbnail, err := thumb.FromFile(fileName, f.FileHash, conf.ThumbnailsPath(), thumbType.Width, thumbType.Height, thumbType.Options...); err == nil {
if c.Query("download") != "" {
downloadFileName := f.DownloadFileName()
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", downloadFileName))
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", f.DownloadFileName()))
}
c.File(thumbnail)

9
internal/api/label.go
View file

@ -163,6 +163,15 @@ func LabelThumbnail(router *gin.RouterGroup, conf *config.Config) {
return
}
// Use original file if thumb size exceeds limit, see https://github.com/photoprism/photoprism/issues/157
if thumbType.Height > thumb.MaxHeight || thumbType.Width > thumb.MaxWidth {
log.Debugf("label: using original, thumbnail size exceeds limit (width %d, height %d)", thumbType.Width, thumbType.Height)
c.File(fileName)
return
}
if thumbnail, err := thumb.FromFile(fileName, f.FileHash, conf.ThumbnailsPath(), thumbType.Width, thumbType.Height, thumbType.Options...); err == nil {
thumbData, err := ioutil.ReadFile(thumbnail)

17
internal/api/photo_thumbnail.go
View file

@ -50,11 +50,22 @@ func GetThumbnail(router *gin.RouterGroup, conf *config.Config) {
return
}
// Use original file if thumb size exceeds limit, see https://github.com/photoprism/photoprism/issues/157
if thumbType.Height > thumb.MaxHeight || thumbType.Width > thumb.MaxWidth {
log.Debugf("photo: using original, thumbnail size exceeds limit (width %d, height %d)", thumbType.Width, thumbType.Height)
if c.Query("download") != "" {
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", f.DownloadFileName()))
}
c.File(fileName)
return
}
if thumbnail, err := thumb.FromFile(fileName, f.FileHash, conf.ThumbnailsPath(), thumbType.Width, thumbType.Height, thumbType.Options...); err == nil {
if c.Query("download") != "" {
downloadFileName := f.DownloadFileName()
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", downloadFileName))
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", f.DownloadFileName()))
}
c.File(thumbnail)

4
internal/config/db.go
View file

@ -11,6 +11,7 @@ import (
_ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/tidb"
)
@ -86,6 +87,9 @@ func (c *Config) MigrateDb() {
// When used with the internal driver, it may create a new database server instance.
// It tries to do this 12 times with a 5 second sleep interval in between.
func (c *Config) connectToDatabase(ctx context.Context) error {
mutex.Db.Lock()
defer mutex.Db.Unlock()
dbDriver := c.DatabaseDriver()
dbDsn := c.DatabaseDsn()

10
internal/config/flags.go
View file

@ -17,7 +17,7 @@ var GlobalFlags = []cli.Flag{
EnvVar: "PHOTOPRISM_READ_ONLY",
},
cli.BoolFlag{
Name: "public",
Name: "public, p",
Usage: "no authentication required",
EnvVar: "PHOTOPRISM_PUBLIC",
},
@ -175,12 +175,12 @@ var GlobalFlags = []cli.Flag{
EnvVar: "PHOTOPRISM_HEIFCONVERT_BIN",
},
cli.IntFlag{
Name: "http-port, p",
Name: "http-port",
Usage: "HTTP server port",
EnvVar: "PHOTOPRISM_HTTP_PORT",
},
cli.StringFlag{
Name: "http-host, i",
Name: "http-host",
Usage: "HTTP server host",
EnvVar: "PHOTOPRISM_HTTP_HOST",
},
@ -190,7 +190,7 @@ var GlobalFlags = []cli.Flag{
EnvVar: "PHOTOPRISM_HTTP_MODE",
},
cli.IntFlag{
Name: "sql-port, s",
Name: "sql-port",
Usage: "built-in SQL server port",
EnvVar: "PHOTOPRISM_SQL_PORT",
},
@ -237,7 +237,7 @@ var GlobalFlags = []cli.Flag{
EnvVar: "PHOTOPRISM_THUMB_QUALITY",
},
cli.IntFlag{
Name: "thumb-size",
Name: "thumb-size, s",
Usage: "max thumbnail size in pixels (720-16384)",
Value: 8192,
EnvVar: "PHOTOPRISM_THUMB_SIZE",

6
internal/entity/camera.go
View file

@ -7,6 +7,7 @@ import (
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Camera model and make (as extracted from UpdateExif metadata)
@ -51,8 +52,9 @@ func NewCamera(modelName string, makeName string) *Camera {
}
func (m *Camera) FirstOrCreate(db *gorm.DB) *Camera {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "camera_model = ? AND camera_make = ?", m.CameraModel, m.CameraMake).Error; err != nil {
log.Errorf("camera: %s", err)
}

6
internal/entity/country.go
View file

@ -4,6 +4,7 @@ import (
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/maps"
"github.com/photoprism/photoprism/internal/mutex"
)
var altCountryNames = map[string]string{
@ -51,8 +52,9 @@ func NewCountry(countryCode string, countryName string) *Country {
}
func (m *Country) FirstOrCreate(db *gorm.DB) *Country {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "id = ?", m.ID).Error; err != nil {
log.Errorf("country: %s", err)
}

2
internal/entity/entity.go
View file

@ -11,7 +11,6 @@ package entity
import (
"strconv"
"sync"
"time"
"github.com/jinzhu/gorm"
@ -20,7 +19,6 @@ import (
)
var log = event.Log
var writeMutex = sync.Mutex{}
func logError(result *gorm.DB) {
if result.Error != nil {

6
internal/entity/keyword.go
View file

@ -4,6 +4,7 @@ import (
"strings"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Keyword for full text search
@ -24,8 +25,9 @@ func NewKeyword(keyword string) *Keyword {
}
func (m *Keyword) FirstOrCreate(db *gorm.DB) *Keyword {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "keyword = ?", m.Keyword).Error; err != nil {
log.Errorf("keyword: %s", err)
}

6
internal/entity/label.go
View file

@ -6,6 +6,7 @@ import (
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Labels for photo, album and location categorization
@ -53,8 +54,9 @@ func NewLabel(labelName string, labelPriority int) *Label {
}
func (m *Label) FirstOrCreate(db *gorm.DB) *Label {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "label_slug = ?", m.LabelSlug).Error; err != nil {
log.Errorf("label: %s", err)
}

6
internal/entity/lens.go
View file

@ -6,6 +6,7 @@ import (
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Camera lens (as extracted from UpdateExif metadata)
@ -47,8 +48,9 @@ func NewLens(modelName string, makeName string) *Lens {
}
func (m *Lens) FirstOrCreate(db *gorm.DB) *Lens {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "lens_slug = ?", m.LensSlug).Error; err != nil {
log.Errorf("lens: %s", err)
}

5
internal/entity/location.go
View file

@ -7,6 +7,7 @@ import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/maps"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/s2"
"github.com/photoprism/photoprism/internal/txt"
)
@ -45,8 +46,8 @@ func NewLocation(lat, lng float64) *Location {
}
func (m *Location) Find(db *gorm.DB, api string) error {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.First(m, "id = ?", m.ID).Error; err == nil {
m.Place = FindPlace(m.PlaceID, db)

2
internal/entity/photo.go
View file

@ -30,7 +30,7 @@ type Photo struct {
PhotoFocalLength int
PhotoIso int
PhotoFNumber float64
PhotoExposure string `gorm:"type:varbinary(16);"`
PhotoExposure string `gorm:"type:varbinary(32);"`
PhotoViews uint
Camera *Camera
CameraID uint `gorm:"index:idx_photos_camera_lens;"`

6
internal/entity/photo_album.go
View file

@ -4,6 +4,7 @@ import (
"time"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Photos can be added to multiple albums
@ -31,8 +32,9 @@ func NewPhotoAlbum(photoUUID, albumUUID string) *PhotoAlbum {
}
func (m *PhotoAlbum) FirstOrCreate(db *gorm.DB) *PhotoAlbum {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "photo_uuid = ? AND album_uuid = ?", m.PhotoUUID, m.AlbumUUID).Error; err != nil {
log.Errorf("photo album: %s", err)
}

10
internal/entity/photo_keyword.go
View file

@ -1,6 +1,9 @@
package entity
import "github.com/jinzhu/gorm"
import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
type PhotoKeyword struct {
PhotoID uint `gorm:"primary_key;auto_increment:false"`
@ -21,8 +24,9 @@ func NewPhotoKeyword(photoID, keywordID uint) *PhotoKeyword {
}
func (m *PhotoKeyword) FirstOrCreate(db *gorm.DB) *PhotoKeyword {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "photo_id = ? AND keyword_id = ?", m.PhotoID, m.KeywordID).Error; err != nil {
log.Errorf("photo keyword: %s", err)
}

6
internal/entity/photo_label.go
View file

@ -2,6 +2,7 @@ package entity
import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
// Photo labels are weighted by uncertainty (100 - confidence)
@ -30,8 +31,9 @@ func NewPhotoLabel(photoId, labelId uint, uncertainty int, source string) *Photo
}
func (m *PhotoLabel) FirstOrCreate(db *gorm.DB) *PhotoLabel {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "photo_id = ? AND label_id = ?", m.PhotoID, m.LabelID).Error; err != nil {
log.Errorf("photo label: %s", err)
}

6
internal/entity/place.go
View file

@ -5,6 +5,7 @@ import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/maps"
"github.com/photoprism/photoprism/internal/mutex"
)
// Photo place
@ -72,8 +73,9 @@ func (m *Place) Find(db *gorm.DB) error {
}
func (m *Place) FirstOrCreate(db *gorm.DB) *Place {
writeMutex.Lock()
defer writeMutex.Unlock()
mutex.Db.Lock()
defer mutex.Db.Unlock()
if err := db.FirstOrCreate(m, "id = ? OR loc_label = ?", m.ID, m.LocLabel).Error; err != nil {
log.Debugf("place: %s for token %s or label \"%s\"", err.Error(), m.ID, m.LocLabel)
}

61
internal/mutex/busy.go Normal file
View file

@ -0,0 +1,61 @@
package mutex
import (
"errors"
"sync"
)
type Busy struct {
busy bool
canceled bool
mutex sync.Mutex
}
func (b *Busy) Busy() bool {
b.mutex.Lock()
defer b.mutex.Unlock()
return b.busy
}
func (b *Busy) Start() error {
b.mutex.Lock()
defer b.mutex.Unlock()
if b.canceled {
return errors.New("still running")
}
if b.busy {
return errors.New("already running")
}
b.busy = true
b.canceled = false
return nil
}
func (b *Busy) Stop() {
b.mutex.Lock()
defer b.mutex.Unlock()
b.busy = false
b.canceled = false
}
func (b *Busy) Cancel() {
b.mutex.Lock()
defer b.mutex.Unlock()
if b.busy {
b.canceled = true
}
}
func (b *Busy) Canceled() bool {
b.mutex.Lock()
defer b.mutex.Unlock()
return b.canceled
}

23
internal/mutex/busy_test.go Normal file
View file

@ -0,0 +1,23 @@
package mutex
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBusy_Busy(t *testing.T) {
b := Busy{}
assert.False(t, b.Busy())
assert.False(t, b.Canceled())
assert.Nil(t, b.Start())
assert.True(t, b.Busy())
assert.False(t, b.Canceled())
b.Cancel()
assert.True(t, b.Canceled())
assert.True(t, b.Busy())
b.Stop()
assert.False(t, b.Canceled())
assert.False(t, b.Busy())
}

9
internal/mutex/mutex.go Normal file
View file

@ -0,0 +1,9 @@
package mutex
import (
"sync"
)
var Db = sync.Mutex{}
var Worker = Busy{}

1
internal/photoprism/convert.go
View file

@ -24,7 +24,6 @@ func NewConvert(conf *config.Config) *Convert {
// Path converts all files in a directory to JPEG if possible.
func (c *Convert) Path(path string) {
err := filepath.Walk(path, func(filename string, fileInfo os.FileInfo, err error) error {
if err != nil {
log.Error("Walk", err.Error())
return nil

23
internal/photoprism/import.go
View file

@ -13,6 +13,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/file"
"github.com/photoprism/photoprism/internal/mutex"
)
// Import represents an importer that can copy/move MediaFiles to the originals directory.
@ -49,18 +50,12 @@ func (imp *Import) Start(importPath string) {
done := make(map[string]bool)
ind := imp.index
if ind.running {
event.Error("index already running")
if err := mutex.Worker.Start(); err != nil {
event.Error(fmt.Sprintf("import: %s", err.Error()))
return
}
ind.running = true
ind.canceled = false
defer func() {
ind.running = false
ind.canceled = false
}()
defer mutex.Worker.Stop()
if err := ind.tensorFlow.Init(); err != nil {
log.Errorf("import: %s", err.Error())
@ -89,7 +84,7 @@ func (imp *Import) Start(importPath string) {
}
}()
if ind.canceled {
if mutex.Worker.Canceled() {
return errors.New("importing canceled")
}
@ -180,7 +175,7 @@ func (imp *Import) Start(importPath string) {
// Cancel stops the current import operation.
func (imp *Import) Cancel() {
imp.index.Cancel()
mutex.Worker.Cancel()
}
// DestinationFilename returns the destination filename of a MediaFile to be imported.
@ -189,9 +184,9 @@ func (imp *Import) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile
fileExtension := mediaFile.Extension()
dateCreated := mainFile.DateCreated()
if file, err := entity.FindFileByHash(imp.conf.Db(), mediaFile.Hash()); err == nil {
existingFilename := imp.conf.OriginalsPath() + string(os.PathSeparator) + file.FileName
return existingFilename, fmt.Errorf("\"%s\" is identical to \"%s\" (%s)", mediaFile.Filename(), file.FileName, mediaFile.Hash())
if f, err := entity.FindFileByHash(imp.conf.Db(), mediaFile.Hash()); err == nil {
existingFilename := imp.conf.OriginalsPath() + string(os.PathSeparator) + f.FileName
return existingFilename, fmt.Errorf("\"%s\" is identical to \"%s\" (%s)", mediaFile.Filename(), f.FileName, mediaFile.Hash())
}
// Mon Jan 2 15:04:05 -0700 MST 2006

20
internal/photoprism/index.go
View file

@ -2,6 +2,7 @@ package photoprism
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@ -10,6 +11,7 @@ import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/nsfw"
)
@ -19,8 +21,6 @@ type Index struct {
tensorFlow *TensorFlow
nsfwDetector *nsfw.Detector
db *gorm.DB
running bool
canceled bool
}
// NewIndex returns a new indexer and expects its dependencies as arguments.
@ -45,25 +45,19 @@ func (ind *Index) thumbnailsPath() string {
// Cancel stops the current indexing operation.
func (ind *Index) Cancel() {
ind.canceled = true
mutex.Worker.Cancel()
}
// Start will index MediaFiles in the originals directory.
func (ind *Index) Start(options IndexOptions) map[string]bool {
done := make(map[string]bool)
if ind.running {
event.Error("index already running")
if err := mutex.Worker.Start(); err != nil {
event.Error(fmt.Sprintf("index: %s", err.Error()))
return done
}
ind.running = true
ind.canceled = false
defer func() {
ind.running = false
ind.canceled = false
}()
defer mutex.Worker.Stop()
if err := ind.tensorFlow.Init(); err != nil {
log.Errorf("index: %s", err.Error())
@ -91,7 +85,7 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
}
}()
if ind.canceled {
if mutex.Worker.Canceled() {
return errors.New("indexing canceled")
}

2
internal/photoprism/thumbnails.go
View file

@ -101,7 +101,7 @@ func (m *MediaFile) CreateDefaultThumbnails(thumbPath string, force bool) (err e
thumbType := thumb.Types[name]
if thumbType.Height > thumb.MaxHeight || thumbType.Width > thumb.MaxWidth {
log.Debugf("thumbs: size exceeds limit (width %d, height %d)", thumbType.Width, thumbType.Height)
// Skip, size exceeds limit
continue
}

4
internal/photoprism/thumbnails_test.go
View file

@ -140,11 +140,11 @@ func TestThumbnails_Filename(t *testing.T) {
})
t.Run("invalid width", func(t *testing.T) {
_, err := thumb.Filename("99988", thumbsPath, -4, 150, thumb.ResampleFit, thumb.ResampleNearestNeighbor)
assert.Equal(t, "thumbs: width has an invalid value (-4)", err.Error())
assert.Equal(t, "thumbs: width exceeds limit (-4)", err.Error())
})
t.Run("invalid height", func(t *testing.T) {
_, err := thumb.Filename("99988", thumbsPath, 200, -1, thumb.ResampleFit, thumb.ResampleNearestNeighbor)
assert.Equal(t, "thumbs: height has an invalid value (-1)", err.Error())
assert.Equal(t, "thumbs: height exceeds limit (-1)", err.Error())
})
t.Run("empty thumbpath", func(t *testing.T) {
path := ""

4
internal/thumb/create.go
View file

@ -73,11 +73,11 @@ func Postfix(width, height int, opts ...ResampleOption) (result string) {
func Filename(hash string, thumbPath string, width, height int, opts ...ResampleOption) (filename string, err error) {
if width < 0 || width > MaxWidth {
return "", fmt.Errorf("thumbs: width has an invalid value (%d)", width)
return "", fmt.Errorf("thumbs: width exceeds limit (%d)", width)
}
if height < 0 || height > MaxHeight {
return "", fmt.Errorf("thumbs: height has an invalid value (%d)", height)
return "", fmt.Errorf("thumbs: height exceeds limit (%d)", height)
}
if len(hash) < 4 {