beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Config: Add option to limit originals size in megapixels #1017

Browse source 
Adds the `megapixel-limit` and `thumb-colorspace` config options.
This commit is contained in:
Michael Mayer 2022-04-01 21:14:22 +02:00
parent 557dc24e1b
commit 728cb2144c
14 changed files with 220 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
internal/commands/config.go
View file

@ -39,9 +39,12 @@ func configAction(ctx *cli.Context) error {
fmt.Printf("%-25s %s\n", "config-path", conf.ConfigPath())
fmt.Printf("%-25s %s\n", "settings-file", conf.SettingsFile())
// Paths.
// Originals.
fmt.Printf("%-25s %s\n", "originals-path", conf.OriginalsPath())
fmt.Printf("%-25s %d\n", "originals-limit", conf.OriginalsLimit())
fmt.Printf("%-25s %d\n", "megapixel-limit", conf.MegapixelLimit())
// Other paths.
fmt.Printf("%-25s %s\n", "storage-path", conf.StoragePath())
fmt.Printf("%-25s %s\n", "import-path", conf.ImportPath())
fmt.Printf("%-25s %s\n", "cache-path", conf.CachePath())
@ -149,6 +152,7 @@ func configAction(ctx *cli.Context) error {
fmt.Printf("%-25s %s\n", "download-token", conf.DownloadToken())
fmt.Printf("%-25s %s\n", "preview-token", conf.PreviewToken())
fmt.Printf("%-25s %s\n", "thumb-filter", conf.ThumbFilter())
fmt.Printf("%-25s %s\n", "thumb-colorspace", conf.ThumbColorspace())
fmt.Printf("%-25s %t\n", "thumb-uncached", conf.ThumbUncached())
fmt.Printf("%-25s %d\n", "thumb-size", conf.ThumbSizePrecached())
fmt.Printf("%-25s %d\n", "thumb-size-uncached", conf.ThumbSizeUncached())

30
internal/config/config.go
View file

@ -148,6 +148,7 @@ func (c *Config) Propagate() {
log.SetLevel(c.LogLevel())
// Set thumbnail generation parameters.
thumb.StandardRGB = c.ThumbStandardRGB()
thumb.SizePrecached = c.ThumbSizePrecached()
thumb.SizeUncached = c.ThumbSizeUncached()
thumb.Filter = c.ThumbFilter()
@ -577,14 +578,37 @@ func (c *Config) GeoApi() string {
return "places"
}
// OriginalsLimit returns the file size limit for originals.
// OriginalsLimit returns the maximum size of originals in megabytes.
func (c *Config) OriginalsLimit() int64 {
if c.options.OriginalsLimit <= 0 || c.options.OriginalsLimit > 100000 {
return -1
}
// Megabyte.
return c.options.OriginalsLimit * 1024 * 1024
return c.options.OriginalsLimit
}
// OriginalsLimitBytes returns the maximum size of originals in bytes.
func (c *Config) OriginalsLimitBytes() int64 {
if megabyte := c.OriginalsLimit(); megabyte < 1 {
return -1
} else {
return megabyte * 1024 * 1024
}
}
// MegapixelLimit returns the maximum resolution of originals in megapixels (width x height).
func (c *Config) MegapixelLimit() int {
mp := c.options.MegapixelLimit
if mp < 0 {
return -1
} else if c.options.MegapixelLimit > 900 {
mp = 900
} else if c.options.MegapixelLimit == 0 {
mp = 100
}
return mp
}
// UpdateHub updates backend api credentials for maps & places.

22
internal/config/config_test.go
View file

@ -308,7 +308,27 @@ func TestConfig_OriginalsLimit(t *testing.T) {
assert.Equal(t, int64(-1), c.OriginalsLimit())
c.options.OriginalsLimit = 800
assert.Equal(t, int64(838860800), c.OriginalsLimit())
assert.Equal(t, int64(800), c.OriginalsLimit())
}
func TestConfig_OriginalsLimitBytes(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, int64(-1), c.OriginalsLimitBytes())
c.options.OriginalsLimit = 800
assert.Equal(t, int64(838860800), c.OriginalsLimitBytes())
}
func TestConfig_MegapixelLimit(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, 100, c.MegapixelLimit())
c.options.MegapixelLimit = 800
assert.Equal(t, 800, c.MegapixelLimit())
c.options.MegapixelLimit = 950
assert.Equal(t, 900, c.MegapixelLimit())
c.options.MegapixelLimit = -1
assert.Equal(t, -1, c.MegapixelLimit())
}
func TestConfig_BaseUri(t *testing.T) {

18
internal/config/flags.go
View file

@ -89,9 +89,15 @@ var GlobalFlags = []cli.Flag{
cli.IntFlag{
Name: "originals-limit",
Value: 1000,
Usage: "file size limit in `MB`",
Usage: "maximum size of media files in `MEGABYTES` (1-100000; -1 to disable)",
EnvVar: "PHOTOPRISM_ORIGINALS_LIMIT",
},
cli.IntFlag{
Name: "megapixel-limit",
Value: 100,
Usage: "maximum resolution of media files in `MEGAPIXELS` (1-900; -1 to disable)",
EnvVar: "PHOTOPRISM_MEGAPIXEL_LIMIT",
},
cli.StringFlag{
Name: "storage-path, t",
Usage: "writable storage `PATH` for cache, database, and sidecar files",
@ -141,13 +147,13 @@ var GlobalFlags = []cli.Flag{
},
cli.IntFlag{
Name: "auto-index",
Usage: "WebDAV auto index safety delay in `SECONDS`, disable with -1",
Usage: "WebDAV auto index safety delay in `SECONDS` (-1 to disable)",
Value: DefaultAutoIndexDelay,
EnvVar: "PHOTOPRISM_AUTO_INDEX",
},
cli.IntFlag{
Name: "auto-import",
Usage: "WebDAV auto import safety delay in `SECONDS`, disable with -1",
Usage: "WebDAV auto import safety delay in `SECONDS` (-1 to disable)",
Value: DefaultAutoImportDelay,
EnvVar: "PHOTOPRISM_AUTO_IMPORT",
},
@ -461,6 +467,12 @@ var GlobalFlags = []cli.Flag{
Value: "lanczos",
EnvVar: "PHOTOPRISM_THUMB_FILTER",
},
cli.StringFlag{
Name: "thumb-colorspace",
Usage: "convert Apple Display P3 colors in thumbnails to standard color space",
Value: "sRGB",
EnvVar: "PHOTOPRISM_THUMB_COLORSPACE",
},
cli.BoolFlag{
Name: "thumb-uncached, u",
Usage: "enable on-demand creation of missing thumbnails (high memory and cpu usage)",

2
internal/config/options.go
View file

@ -52,6 +52,7 @@ type Options struct {
ConfigFile string `json:"-"`
OriginalsPath string `yaml:"OriginalsPath" json:"-" flag:"originals-path"`
OriginalsLimit int64 `yaml:"OriginalsLimit" json:"OriginalsLimit" flag:"originals-limit"`
MegapixelLimit int `yaml:"MegapixelLimit" json:"MegapixelLimit" flag:"megapixel-limit"`
StoragePath string `yaml:"StoragePath" json:"-" flag:"storage-path"`
ImportPath string `yaml:"ImportPath" json:"-" flag:"import-path"`
CachePath string `yaml:"CachePath" json:"-" flag:"cache-path"`
@ -121,6 +122,7 @@ type Options struct {
DownloadToken string `yaml:"DownloadToken" json:"-" flag:"download-token"`
PreviewToken string `yaml:"PreviewToken" json:"-" flag:"preview-token"`
ThumbFilter string `yaml:"ThumbFilter" json:"ThumbFilter" flag:"thumb-filter"`
ThumbColorspace string `yaml:"ThumbColorspace" json:"ThumbColorspace" flag:"thumb-colorspace"`
ThumbUncached bool `yaml:"ThumbUncached" json:"ThumbUncached" flag:"thumb-uncached"`
ThumbSize int `yaml:"ThumbSize" json:"ThumbSize" flag:"thumb-size"`
ThumbSizeUncached int `yaml:"ThumbSizeUncached" json:"ThumbSizeUncached" flag:"thumb-size-uncached"`

12
internal/config/resample.go
View file

@ -38,11 +38,21 @@ func (c *Config) ThumbFilter() thumb.ResampleFilter {
}
}
// ThumbPath returns the thumbnails directory.
// ThumbPath returns the thumbnail storage directory.
func (c *Config) ThumbPath() string {
return c.CachePath() + "/thumbnails"
}
// ThumbColorspace returns the standard colorspace for thumbnails.
func (c *Config) ThumbColorspace() string {
return c.options.ThumbColorspace
}
// ThumbStandardRGB checks if colors should be normalized to standard RGB in thumbnails.
func (c *Config) ThumbStandardRGB() bool {
return strings.ToLower(c.ThumbColorspace()) == "srgb"
}
// ThumbUncached checks if on-demand thumbnail rendering is enabled (high memory and cpu usage).
func (c *Config) ThumbUncached() bool {
return c.options.ThumbUncached

11
internal/photoprism/import_worker.go
View file

@ -158,15 +158,16 @@ func ImportWorker(jobs <-chan ImportJob) {
done := make(map[string]bool)
ind := imp.index
sizeLimit := ind.conf.OriginalsLimit()
limitSize := ind.conf.OriginalsLimitBytes()
photoUID := ""
if related.Main != nil {
f := related.Main
// Enforce file size limit for originals.
if sizeLimit > 0 && f.FileSize() > sizeLimit {
log.Warnf("import: %s exceeds file size limit (%d / %d MB)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), sizeLimit/(1024*1024))
if limitSize > 0 && f.FileSize() > limitSize {
log.Warnf("import: %s exceeds file size limit (%d / %d megabyte)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), limitSize/(1024*1024))
continue
}
@ -200,8 +201,8 @@ func ImportWorker(jobs <-chan ImportJob) {
done[f.FileName()] = true
// Enforce file size limit for originals.
if sizeLimit > 0 && f.FileSize() > sizeLimit {
log.Warnf("import: %s exceeds file size limit (%d / %d MB)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), sizeLimit/(1024*1024))
if limitSize > 0 && f.FileSize() > limitSize {
log.Warnf("import: %s exceeds file size limit (%d / %d megabyte)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), limitSize/(1024*1024))
continue
}

7
internal/photoprism/index_mediafile.go
View file

@ -130,6 +130,13 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName, photoUID
}
}
// Skip media files that exceed the configured resolution limit, unless the file has already been indexed.
if !fileExists && m.ExceedsMegapixelLimit() {
log.Warnf("index: %s exceeds resolution limit (%d / %d megapixels)", logName, m.Megapixels(), Config().MegapixelLimit())
result.Status = IndexSkipped
return result
}
// Find existing photo if a photo uid was provided or file has not been indexed yet...
if !fileExists && photoUID != "" {
// Find existing photo by UID.

3
internal/photoprism/index_mediafile_test.go
View file

@ -40,6 +40,9 @@ func TestIndex_MediaFile(t *testing.T) {
words := mediaFile.metaData.Keywords.String()
t.Logf("size in megapixel: %d", mediaFile.Megapixels())
t.Logf("megapixel limit exceeded: %t", mediaFile.ExceedsMegapixelLimit())
assert.Contains(t, words, "marienkäfer")
assert.Contains(t, words, "burst")
assert.Contains(t, words, "flash")

10
internal/photoprism/index_related.go
View file

@ -21,11 +21,11 @@ func IndexMain(related *RelatedFiles, ind *Index, opt IndexOptions) (result Inde
}
f := related.Main
sizeLimit := ind.conf.OriginalsLimit()
limitSize := ind.conf.OriginalsLimitBytes()
// Enforce file size limit for originals.
if sizeLimit > 0 && f.FileSize() > sizeLimit {
result.Err = fmt.Errorf("index: %s exceeds file size limit (%d / %d MB)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), sizeLimit/(1024*1024))
if limitSize > 0 && f.FileSize() > limitSize {
result.Err = fmt.Errorf("index: %s exceeds file size limit (%d / %d megabyte)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), limitSize/(1024*1024))
result.Status = IndexFailed
return result
}
@ -82,7 +82,7 @@ func IndexRelated(related RelatedFiles, ind *Index, opt IndexOptions) (result In
}
done := make(map[string]bool)
sizeLimit := ind.conf.OriginalsLimit()
sizeLimit := ind.conf.OriginalsLimitBytes()
result = IndexMain(&related, ind, opt)
@ -123,7 +123,7 @@ func IndexRelated(related RelatedFiles, ind *Index, opt IndexOptions) (result In
// Enforce file size limit for originals.
if sizeLimit > 0 && f.FileSize() > sizeLimit {
log.Warnf("index: %s exceeds file size limit (%d / %d MB)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), sizeLimit/(1024*1024))
log.Warnf("index: %s exceeds file size limit (%d / %d megabyte)", sanitize.Log(f.BaseName()), f.FileSize()/(1024*1024), sizeLimit/(1024*1024))
continue
}

41
internal/photoprism/mediafile.go
View file

@ -744,17 +744,17 @@ func (m *MediaFile) IsSidecar() bool {
return m.MediaType() == fs.MediaSidecar
}
// IsPlayableVideo returns true if this is a supported video file format.
// IsPlayableVideo checks if the file is a video in playable format.
func (m *MediaFile) IsPlayableVideo() bool {
return m.IsVideo() && (m.HasFileType(fs.FormatMp4) || m.HasFileType(fs.FormatAvc))
}
// IsPhoto returns true if this file is a photo / image.
func (m *MediaFile) IsPhoto() bool {
// IsImage checks if the file is an image
func (m *MediaFile) IsImage() bool {
return m.IsJpeg() || m.IsRaw() || m.IsHEIF() || m.IsImageOther()
}
// IsLive returns true if this is a live photo.
// IsLive checks if the file is a live photo.
func (m *MediaFile) IsLive() bool {
if m.IsHEIF() {
return fs.FormatMov.FindFirst(m.FileName(), []string{}, Config().OriginalsPath(), false) != ""
@ -906,9 +906,34 @@ func (m *MediaFile) Portrait() bool {
return m.Width() < m.Height()
}
// Megapixels returns the resolution in megapixels.
func (m *MediaFile) Megapixels() int {
return int(math.Round(float64(m.Width()*m.Height()) / 1000000))
// Megapixels returns the resolution in megapixels if possible.
func (m *MediaFile) Megapixels() (resolution int) {
if !m.IsMedia() {
return 0
}
if m.IsJpeg() || m.IsPng() || m.IsGif() {
resolution = int(math.Round(float64(m.Width()*m.Height()) / 1000000))
}
if resolution <= 0 {
resolution = m.MetaData().Megapixels()
}
return resolution
}
// ExceedsMegapixelLimit checks if the media file exceeds the configured resolution limit in megapixels.
func (m *MediaFile) ExceedsMegapixelLimit() bool {
if !m.IsMedia() {
return false
} else if limit := Config().MegapixelLimit(); limit <= 0 {
return false
} else if mp := m.Megapixels(); mp <= 0 {
return false
} else {
return mp > limit
}
}
// Orientation returns the Exif orientation of the media file.
@ -951,7 +976,7 @@ func (m *MediaFile) Resample(path string, sizeName thumb.Name) (img image.Image,
return imaging.Open(filename)
}
// ResampleDefault pre-caches default thumbnails.
// ResampleDefault creates the configured default thumbnails at indexing time.
func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
count := 0
start := time.Now()

18
internal/photoprism/mediafile_test.go
View file

@ -1347,7 +1347,7 @@ func TestMediaFile_IsSidecar(t *testing.T) {
})
}
func TestMediaFile_IsPhoto(t *testing.T) {
func TestMediaFile_IsImage(t *testing.T) {
t.Run("iphone_7.json", func(t *testing.T) {
conf := config.TestConfig()
@ -1355,14 +1355,14 @@ func TestMediaFile_IsPhoto(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assert.Equal(t, false, mediaFile.IsPhoto())
assert.Equal(t, false, mediaFile.IsImage())
})
t.Run("iphone_7.xmp", func(t *testing.T) {
conf := config.TestConfig()
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/iphone_7.xmp")
assert.Nil(t, err)
assert.Equal(t, false, mediaFile.IsPhoto())
assert.Equal(t, false, mediaFile.IsImage())
})
t.Run("iphone_7.heic", func(t *testing.T) {
conf := config.TestConfig()
@ -1371,7 +1371,7 @@ func TestMediaFile_IsPhoto(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assert.Equal(t, true, mediaFile.IsPhoto())
assert.Equal(t, true, mediaFile.IsImage())
})
t.Run("canon_eos_6d.dng", func(t *testing.T) {
conf := config.TestConfig()
@ -1380,14 +1380,14 @@ func TestMediaFile_IsPhoto(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assert.Equal(t, true, mediaFile.IsPhoto())
assert.Equal(t, true, mediaFile.IsImage())
})
t.Run("elephants.jpg", func(t *testing.T) {
conf := config.TestConfig()
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/elephants.jpg")
assert.Nil(t, err)
assert.Equal(t, true, mediaFile.IsPhoto())
assert.Equal(t, true, mediaFile.IsImage())
})
}
@ -1398,7 +1398,7 @@ func TestMediaFile_IsVideo(t *testing.T) {
if f, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "christmas.mp4")); err != nil {
t.Fatal(err)
} else {
assert.Equal(t, false, f.IsPhoto())
assert.Equal(t, false, f.IsImage())
assert.Equal(t, true, f.IsVideo())
assert.Equal(t, false, f.IsJson())
assert.Equal(t, false, f.IsSidecar())
@ -1408,7 +1408,7 @@ func TestMediaFile_IsVideo(t *testing.T) {
if f, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "canon_eos_6d.dng")); err != nil {
t.Fatal(err)
} else {
assert.Equal(t, true, f.IsPhoto())
assert.Equal(t, true, f.IsImage())
assert.Equal(t, false, f.IsVideo())
assert.Equal(t, false, f.IsJson())
assert.Equal(t, false, f.IsSidecar())
@ -1418,7 +1418,7 @@ func TestMediaFile_IsVideo(t *testing.T) {
if f, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "iphone_7.json")); err != nil {
t.Fatal(err)
} else {
assert.Equal(t, false, f.IsPhoto())
assert.Equal(t, false, f.IsImage())
assert.Equal(t, false, f.IsVideo())
assert.Equal(t, true, f.IsJson())
assert.Equal(t, true, f.IsSidecar())

66
internal/thumb/open.go
View file

@ -6,17 +6,14 @@ import (
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"os"
"path/filepath"
"github.com/disintegration/imaging"
"github.com/mandykoh/prism/meta/autometa"
"github.com/photoprism/photoprism/pkg/colors"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/sanitize"
)
// StandardRGB configures whether colors in the Apple Display P3 color space should be converted to standard RGB.
var StandardRGB = true
// Open loads an image from disk, rotates it, and converts the color profile if necessary.
func Open(fileName string, orientation int) (result image.Image, err error) {
if fileName == "" {
@ -24,7 +21,7 @@ func Open(fileName string, orientation int) (result image.Image, err error) {
}
// Open JPEG?
if fs.GetFileFormat(fileName) == fs.FormatJpeg {
if StandardRGB && fs.GetFileFormat(fileName) == fs.FormatJpeg {
return OpenJpeg(fileName, orientation)
}
@ -42,58 +39,3 @@ func Open(fileName string, orientation int) (result image.Image, err error) {
return img, nil
}
// OpenJpeg loads a JPEG image from disk, rotates it, and converts the color profile if necessary.
func OpenJpeg(fileName string, orientation int) (result image.Image, err error) {
if fileName == "" {
return result, fmt.Errorf("filename missing")
}
logName := sanitize.Log(filepath.Base(fileName))
// Open file.
fileReader, err := os.Open(fileName)
if err != nil {
return result, err
}
defer fileReader.Close()
// Read color metadata.
md, imgStream, err := autometa.Load(fileReader)
var img image.Image
if err != nil {
log.Warnf("resample: %s in %s (read color metadata)", err, logName)
img, err = imaging.Decode(fileReader)
} else {
img, err = imaging.Decode(imgStream)
}
if err != nil {
return result, err
}
// Read ICC profile and convert colors if possible.
if md != nil {
if iccProfile, err := md.ICCProfile(); err != nil || iccProfile == nil {
// Do nothing.
log.Tracef("resample: %s has no color profile", logName)
} else if profile, err := iccProfile.Description(); err == nil && profile != "" {
log.Tracef("resample: %s has color profile %s", logName, sanitize.Log(profile))
switch {
case colors.ProfileDisplayP3.Equal(profile):
img = colors.ToSRGB(img, colors.ProfileDisplayP3)
}
}
}
// Rotate?
if orientation > 1 {
img = Rotate(img, orientation)
}
return img, nil
}

72
internal/thumb/open_jpeg.go Normal file
View file

@ -0,0 +1,72 @@
package thumb
import (
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"os"
"path/filepath"
"github.com/disintegration/imaging"
"github.com/mandykoh/prism/meta/autometa"
"github.com/photoprism/photoprism/pkg/colors"
"github.com/photoprism/photoprism/pkg/sanitize"
)
// OpenJpeg loads a JPEG image from disk, rotates it, and converts the color profile if necessary.
func OpenJpeg(fileName string, orientation int) (result image.Image, err error) {
if fileName == "" {
return result, fmt.Errorf("filename missing")
}
logName := sanitize.Log(filepath.Base(fileName))
// Open file.
fileReader, err := os.Open(fileName)
if err != nil {
return result, err
}
defer fileReader.Close()
// Read color metadata.
md, imgStream, err := autometa.Load(fileReader)
var img image.Image
if err != nil {
log.Warnf("resample: %s in %s (read color metadata)", err, logName)
img, err = imaging.Decode(fileReader)
} else {
img, err = imaging.Decode(imgStream)
}
if err != nil {
return result, err
}
// Read ICC profile and convert colors if possible.
if md != nil {
if iccProfile, err := md.ICCProfile(); err != nil || iccProfile == nil {
// Do nothing.
log.Tracef("resample: %s has no color profile", logName)
} else if profile, err := iccProfile.Description(); err == nil && profile != "" {
log.Tracef("resample: %s has color profile %s", logName, sanitize.Log(profile))
switch {
case colors.ProfileDisplayP3.Equal(profile):
img = colors.ToSRGB(img, colors.ProfileDisplayP3)
}
}
}
// Rotate?
if orientation > 1 {
img = Rotate(img, orientation)
}
return img, nil
}