beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Backend: Improve log messages

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-07-14 18:00:32 +02:00
parent c027e31304
commit b62af742ae
13 changed files with 77 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
internal/classify/gen.go
View file

@ -34,7 +34,7 @@ func main() {
fileName := "rules.yml"
if !fs.FileExists(fileName) {
log.Panicf("tensorflow: label rules file not found in %s", txt.Quote(filepath.Base(fileName)))
log.Panicf("classify: label rules not found in %s", txt.Quote(filepath.Base(fileName)))
}
yamlConfig, err := ioutil.ReadFile(fileName)
@ -52,7 +52,7 @@ func main() {
for label, rule := range rules {
for _, char := range label {
if unicode.IsUpper(char) {
log.Panicf("label must be lowercase: %s", label)
log.Panicf("classify: %s must be lowercase", label)
}
}

26
internal/classify/tensorflow.go
View file

@ -3,7 +3,7 @@ package classify
import (
"bufio"
"bytes"
"errors"
"fmt"
"image"
"io/ioutil"
"math"
@ -67,15 +67,14 @@ func (t *TensorFlow) Labels(img []byte) (result Labels, err error) {
return nil, err
}
// Make tensor
tensor, err := t.makeTensor(img, "jpeg")
// Create tensor from image.
tensor, err := t.createTensor(img, "jpeg")
if err != nil {
log.Error(err)
return nil, errors.New("invalid image")
return nil, fmt.Errorf("classify: %s (create tensor)", err.Error())
}
// Run inference
// Run inference.
output, err := t.model.Session.Run(
map[tf.Output]*tf.Tensor{
t.model.Graph.Operation("input_1").Output(0): tensor,
@ -86,19 +85,18 @@ func (t *TensorFlow) Labels(img []byte) (result Labels, err error) {
nil)
if err != nil {
log.Error(err)
return result, errors.New("could not run inference")
return result, fmt.Errorf("classify: %s (run inference)", err.Error())
}
if len(output) < 1 {
return result, errors.New("result is empty")
return result, fmt.Errorf("classify: inference failed, no output")
}
// Return best labels
result = t.bestLabels(output[0].Value().([][]float32)[0])
if len(result) > 0 {
log.Tracef("tensorflow: image classified as %+v", result)
log.Tracef("classify: image classified as %+v", result)
}
return result, nil
@ -107,7 +105,7 @@ func (t *TensorFlow) Labels(img []byte) (result Labels, err error) {
func (t *TensorFlow) loadLabels(path string) error {
modelLabels := path + "/labels.txt"
log.Infof("tensorflow: loading classification labels from labels.txt")
log.Infof("classify: loading labels from labels.txt")
// Load labels
f, err := os.Open(modelLabels)
@ -143,7 +141,7 @@ func (t *TensorFlow) loadModel() error {
modelPath := path.Join(t.modelsPath, t.modelName)
log.Infof("tensorflow: loading image classification model from %s", txt.Quote(filepath.Base(modelPath)))
log.Infof("classify: loading model from %s", txt.Quote(filepath.Base(modelPath)))
// Load model
model, err := tf.LoadSavedModel(modelPath, t.modelTags, nil)
@ -204,8 +202,8 @@ func (t *TensorFlow) bestLabels(probabilities []float32) Labels {
}
}
// makeTensor converts bytes jpeg image in a tensor object required as tensorflow model input
func (t *TensorFlow) makeTensor(image []byte, imageFormat string) (*tf.Tensor, error) {
// createTensor converts bytes jpeg image in a tensor object required as tensorflow model input
func (t *TensorFlow) createTensor(image []byte, imageFormat string) (*tf.Tensor, error) {
img, err := imaging.Decode(bytes.NewReader(image), imaging.AutoOrientation(true))
if err != nil {

14
internal/classify/tensorflow_test.go
View file

@ -137,12 +137,7 @@ func TestTensorFlow_Labels(t *testing.T) {
} else {
result, err := tensorFlow.Labels(imageBuffer)
assert.Empty(t, result)
if err != nil {
assert.Contains(t, err.Error(), "invalid image")
} else {
t.Fatal("err should NOT be nil")
}
assert.Error(t, err)
}
})
t.Run("6720px_white.jpg", func(t *testing.T) {
@ -235,7 +230,7 @@ func TestTensorFlow_MakeTensor(t *testing.T) {
t.Fatal(err)
}
result, err := tensorFlow.makeTensor(imageBuffer, "jpeg")
result, err := tensorFlow.createTensor(imageBuffer, "jpeg")
assert.Equal(t, tensorflow.DataType(0x1), result.DataType())
assert.Equal(t, int64(1), result.Shape()[0])
assert.Equal(t, int64(224), result.Shape()[2])
@ -245,9 +240,10 @@ func TestTensorFlow_MakeTensor(t *testing.T) {
imageBuffer, err := ioutil.ReadFile(examplesPath + "/Random.docx")
assert.Nil(t, err)
result, err := tensorFlow.makeTensor(imageBuffer, "jpeg")
result, err := tensorFlow.createTensor(imageBuffer, "jpeg")
assert.Empty(t, result)
assert.Equal(t, "image: unknown format", err.Error())
assert.EqualError(t, err, "image: unknown format")
})
}

6
internal/entity/cell.go
View file

@ -50,7 +50,7 @@ func (m *Cell) Find(api string) error {
db := Db()
if err := db.Preload("Place").First(m, "id = ?", m.ID).Error; err == nil {
log.Infof("location: found %s (%+v)", m.ID, m)
log.Debugf("location: found %s (%+v)", m.ID, m)
return nil
}
@ -97,13 +97,13 @@ func (m *Cell) Find(api string) error {
m.CellCategory = l.Category()
if err := db.Create(m).Error; err == nil {
log.Infof("location: added %s [%s]", m.ID, time.Since(start))
log.Debugf("location: added %s [%s]", m.ID, time.Since(start))
return nil
} else if err := db.Preload("Place").First(m, "id = ?", m.ID).Error; err != nil {
log.Errorf("location: failed adding %s %s [%s]", m.ID, err.Error(), time.Since(start))
return err
} else {
log.Infof("location: found %s after second try [%s]", m.ID, time.Since(start))
log.Debugf("location: found %s after second try [%s]", m.ID, time.Since(start))
}
return nil

4
internal/entity/place.go
View file

@ -45,11 +45,11 @@ func FindPlace(id string, label string) *Place {
if label == "" {
if err := Db().First(place, "id = ?", id).Error; err != nil {
log.Debugf("place: %s for id %s", err.Error(), id)
log.Debugf("place: %s (id %s)", err.Error(), id)
return nil
}
} else if err := Db().First(place, "id = ? OR place_label = ?", id, label).Error; err != nil {
log.Debugf("place: %s for id %s / label %s", err.Error(), id, txt.Quote(label))
log.Debugf("place: %s (id %s, label %s)", err.Error(), id, txt.Quote(label))
return nil
}

23
internal/nsfw/detector.go
View file

@ -2,7 +2,6 @@ package nsfw
import (
"bufio"
"errors"
"fmt"
"io/ioutil"
"os"
@ -51,11 +50,10 @@ func (t *Detector) Labels(img []byte) (result Labels, err error) {
}
// Make tensor
tensor, err := makeTensorFromImage(img, "jpeg")
tensor, err := createTensorFromImage(img, "jpeg")
if err != nil {
log.Error(err)
return result, errors.New("invalid image")
return result, fmt.Errorf("nsfw: %s (create tensor)", err.Error())
}
// Run inference
@ -69,18 +67,17 @@ func (t *Detector) Labels(img []byte) (result Labels, err error) {
nil)
if err != nil {
log.Error(err)
return result, errors.New("could not run inference")
return result, fmt.Errorf("nsfw: %s (run inference)", err.Error())
}
if len(output) < 1 {
return result, errors.New("result is empty")
return result, fmt.Errorf("nsfw: inference failed, no output")
}
// Return best labels
result = t.getLabels(output[0].Value().([][]float32)[0])
log.Debugf("tensorflow: image classified as %+v", result)
log.Debugf("nsfw: image classified as %+v", result)
return result, nil
}
@ -88,7 +85,7 @@ func (t *Detector) Labels(img []byte) (result Labels, err error) {
func (t *Detector) loadLabels(path string) error {
modelLabels := path + "/labels.txt"
log.Infof("tensorflow: loading classification labels from labels.txt")
log.Infof("nsfw: loading labels from labels.txt")
// Load labels
f, err := os.Open(modelLabels)
@ -122,7 +119,7 @@ func (t *Detector) loadModel() error {
return nil
}
log.Infof("tensorflow: loading image classification model from %s", txt.Quote(filepath.Base(t.modelPath)))
log.Infof("nsfw: loading model from %s", txt.Quote(filepath.Base(t.modelPath)))
// Load model
model, err := tf.LoadSavedModel(t.modelPath, t.modelTags, nil)
@ -146,7 +143,7 @@ func (t *Detector) getLabels(p []float32) Labels {
}
}
func makeTransformImageGraph(imageFormat string) (graph *tf.Graph, input, output tf.Output, err error) {
func transformImageGraph(imageFormat string) (graph *tf.Graph, input, output tf.Output, err error) {
const (
H, W = 224, 224
Mean = float32(117)
@ -178,12 +175,12 @@ func makeTransformImageGraph(imageFormat string) (graph *tf.Graph, input, output
return graph, input, output, err
}
func makeTensorFromImage(image []byte, imageFormat string) (*tf.Tensor, error) {
func createTensorFromImage(image []byte, imageFormat string) (*tf.Tensor, error) {
tensor, err := tf.NewTensor(string(image))
if err != nil {
return nil, err
}
graph, input, output, err := makeTransformImageGraph(imageFormat)
graph, input, output, err := transformImageGraph(imageFormat)
if err != nil {
return nil, err
}

8
internal/photoprism/colors.go
View file

@ -1,25 +1,25 @@
package photoprism
import (
"errors"
"fmt"
"image/color"
"math"
"github.com/lucasb-eyer/go-colorful"
"github.com/photoprism/photoprism/pkg/colors"
"github.com/photoprism/photoprism/pkg/txt"
)
// Colors returns the ColorPerception of an image (only JPEG supported).
func (m *MediaFile) Colors(thumbPath string) (perception colors.ColorPerception, err error) {
if !m.IsJpeg() {
return perception, errors.New("no color information: not a JPEG file")
return perception, fmt.Errorf("%s is not a jpeg file (detect colors)", txt.Quote(m.BaseName()))
}
img, err := m.Resample(thumbPath, "colors")
if err != nil {
log.Printf("can't open image: %s", err.Error())
log.Errorf("%s (detect colors)", err.Error())
return perception, err
}

9
internal/photoprism/convert.go
View file

@ -18,6 +18,7 @@ import (
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/thumb"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/txt"
)
// Convert represents a converter that can convert RAW/HEIF images to JPEG.
@ -60,7 +61,7 @@ func (c *Convert) Start(path string) error {
}
ignore.Log = func(fileName string) {
log.Infof(`convert: ignored "%s"`, fs.RelName(fileName, path))
log.Infof("convert: ignoring %s", txt.Quote(filepath.Base(fileName)))
}
err := godirwalk.Walk(path, &godirwalk.Options{
@ -118,7 +119,7 @@ func (c *Convert) ToJson(mf *MediaFile) (*MediaFile, error) {
}
if !c.conf.SidecarWritable() {
return nil, fmt.Errorf("convert: metadata export to json disabled in read only mode (%s)", mf.RelName(c.conf.OriginalsPath()))
return nil, fmt.Errorf("convert: can't create json sidecar file for %s in read only mode", txt.Quote(mf.BaseName()))
}
jsonName = fs.FileName(mf.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), ".json", c.conf.Settings().Index.Sequences)
@ -184,14 +185,14 @@ func (c *Convert) JpegConvertCommand(mf *MediaFile, jpegName string, xmpName str
result = exec.Command(c.conf.DarktableBin(), args...)
} else {
return nil, useMutex, fmt.Errorf("convert: no raw to jpeg converter installed (%s)", mf.BasePrefix(c.conf.Settings().Index.Sequences))
return nil, useMutex, fmt.Errorf("convert: no converter found for %s", txt.Quote(mf.BaseName()))
}
} else if mf.IsVideo() {
result = exec.Command(c.conf.FFmpegBin(), "-i", mf.FileName(), "-ss", "00:00:00.001", "-vframes", "1", jpegName)
} else if mf.IsHEIF() {
result = exec.Command(c.conf.HeifConvertBin(), mf.FileName(), jpegName)
} else {
return nil, useMutex, fmt.Errorf("convert: file type not supported for conversion (%s)", mf.FileType())
return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", mf.FileType(), txt.Quote(mf.BaseName()))
}
return result, useMutex, nil

8
internal/photoprism/convert_worker.go
View file

@ -1,6 +1,10 @@
package photoprism
import "strings"
import (
"strings"
"github.com/photoprism/photoprism/pkg/txt"
)
type ConvertJob struct {
image *MediaFile
@ -11,7 +15,7 @@ func ConvertWorker(jobs <-chan ConvertJob) {
for job := range jobs {
if _, err := job.convert.ToJpeg(job.image); err != nil {
fileName := job.image.RelName(job.convert.conf.OriginalsPath())
log.Errorf("convert: could not create jpeg for %s (%s)", fileName, strings.TrimSpace(err.Error()))
log.Errorf("convert: %s in %s (jpeg)", strings.TrimSpace(err.Error()), txt.Quote(fileName))
}
}
}

16
internal/photoprism/import_worker.go
View file

@ -29,7 +29,7 @@ func ImportWorker(jobs <-chan ImportJob) {
importPath := job.ImportOpt.Path
if related.Main == nil {
log.Warnf("import: no media file found for %s", txt.Quote(fs.RelName(job.FileName, importPath)))
log.Warnf("import: %s belongs to no supported media file", txt.Quote(fs.RelName(job.FileName, importPath)))
continue
}
@ -81,13 +81,13 @@ func ImportWorker(jobs <-chan ImportJob) {
f, err := NewMediaFile(destinationMainFilename)
if err != nil {
log.Errorf("import: could not import %s (%s)", txt.Quote(fs.RelName(destinationMainFilename, imp.originalsPath())), err.Error())
log.Errorf("import: %s in %s", err.Error(), txt.Quote(fs.RelName(destinationMainFilename, imp.originalsPath())))
continue
}
if !f.HasJpeg() {
if jpegFile, err := imp.convert.ToJpeg(f); err != nil {
log.Errorf("import: creating jpeg failed (%s)", err.Error())
log.Errorf("import: %s in %s (convert to jpeg)", err.Error(), txt.Quote(fs.RelName(destinationMainFilename, imp.originalsPath())))
continue
} else {
log.Infof("import: %s created", fs.RelName(jpegFile.FileName(), imp.originalsPath()))
@ -98,14 +98,14 @@ func ImportWorker(jobs <-chan ImportJob) {
log.Error(err)
} else {
if err := jpg.ResampleDefault(imp.thumbPath(), false); err != nil {
log.Errorf("import: could not create default thumbnails (%s)", err.Error())
log.Errorf("import: %s in %s (resample)", err.Error(), txt.Quote(jpg.BaseName()))
continue
}
}
if imp.conf.SidecarJson() && !f.HasJson() {
if jsonFile, err := imp.convert.ToJson(f); err != nil {
log.Errorf("import: creating json sidecar file failed (%s)", err.Error())
log.Errorf("import: %s in %s (create json sidecar)", err.Error(), txt.Quote(f.BaseName()))
} else {
log.Infof("import: %s created", fs.RelName(jsonFile.FileName(), imp.originalsPath()))
}
@ -114,7 +114,7 @@ func ImportWorker(jobs <-chan ImportJob) {
related, err := f.RelatedFiles(imp.conf.Settings().Index.Sequences)
if err != nil {
log.Errorf("import: could not index %s (%s)", txt.Quote(fs.RelName(destinationMainFilename, imp.originalsPath())), err.Error())
log.Errorf("import: %s in %s (find related files)", err.Error(), txt.Quote(fs.RelName(destinationMainFilename, imp.originalsPath())))
continue
}
@ -125,7 +125,7 @@ func ImportWorker(jobs <-chan ImportJob) {
if related.Main != nil {
// Enforce file size limit for originals.
if ind.conf.OriginalsLimit() > 0 && related.Main.FileSize() > ind.conf.OriginalsLimit() {
log.Warnf("import: %s exceeds file size limit for originals [%d / %d MB]", filepath.Base(related.Main.FileName()), related.Main.FileSize()/(1024*1024), ind.conf.OriginalsLimit()/(1024*1024))
log.Warnf("import: %s exceeds file size limit (%d / %d MB)", filepath.Base(related.Main.FileName()), related.Main.FileSize()/(1024*1024), ind.conf.OriginalsLimit()/(1024*1024))
continue
}
@ -142,7 +142,7 @@ func ImportWorker(jobs <-chan ImportJob) {
continue
}
} else {
log.Warnf("import: no main file for %s (conversion to jpeg failed?)", fs.RelName(destinationMainFilename, imp.originalsPath()))
log.Warnf("import: no main file for %s, conversion to jpeg failed?", fs.RelName(destinationMainFilename, imp.originalsPath()))
}
for _, f := range related.Files {

28
internal/photoprism/index_mediafile.go
View file

@ -164,7 +164,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
if yamlName := fs.TypeYaml.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), stripSequence); yamlName != "" {
if err := photo.LoadFromYaml(yamlName); err != nil {
log.Errorf("index: %s (restore from yaml) for %s", err.Error(), logName)
log.Errorf("index: %s in %s (restore from yaml)", err.Error(), logName)
} else if err := photo.Find(); err != nil {
log.Infof("index: data restored from %s", txt.Quote(fs.RelName(yamlName, Config().OriginalsPath())))
} else {
@ -213,7 +213,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
case m.IsJpeg():
// Color information
if p, err := m.Colors(Config().ThumbPath()); err != nil {
log.Errorf("index: %s for %s", err.Error(), logName)
log.Errorf("index: %s in %s (detect colors)", err.Error(), logName)
} else {
file.FileMainColor = p.MainColor.Name()
file.FileColors = p.Colors.Hex()
@ -513,14 +513,14 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
if photoExists {
if err := photo.Save(); err != nil {
log.Errorf("index: %s for %s", err.Error(), logName)
log.Errorf("index: %s in %s (save photo)", err.Error(), logName)
result.Status = IndexFailed
result.Err = err
return result
}
} else {
if err := photo.FirstOrCreate(); err != nil {
log.Errorf("index: %s", err)
log.Errorf("index: %s in %s (create photo)", err.Error(), logName)
result.Status = IndexFailed
result.Err = err
return result
@ -558,7 +558,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
labels := photo.ClassifyLabels()
if err := photo.UpdateTitle(labels); err != nil {
log.Debugf("%s (%s)", err.Error(), logName)
log.Debugf("%s in %s (update title)", err.Error(), logName)
}
w := txt.Keywords(details.Keywords)
@ -584,18 +584,18 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
photo.PhotoQuality = photo.QualityScore()
if err := photo.Save(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (save photo)", err, logName)
result.Status = IndexFailed
result.Err = err
return result
}
if err := photo.SyncKeywordLabels(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (sync keywords and labels)", err, logName)
}
if err := photo.IndexKeywords(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (save keywords)", err, logName)
}
} else {
if photo.PhotoQuality >= 0 {
@ -603,7 +603,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
}
if err := photo.Save(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (save photo)", err, logName)
result.Status = IndexFailed
result.Err = err
return result
@ -616,7 +616,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
file.UpdatedIn = int64(time.Since(start))
if err := file.Save(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (save file)", err, logName)
result.Status = IndexFailed
result.Err = err
return result
@ -625,7 +625,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
file.CreatedIn = int64(time.Since(start))
if err := file.Create(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (create file)", err, logName)
result.Status = IndexFailed
result.Err = err
return result
@ -644,7 +644,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
if (photo.PhotoType == entity.TypeVideo || photo.PhotoType == entity.TypeLive) && file.FilePrimary {
if err := file.UpdateVideoInfos(); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (update video infos)", err, logName)
}
}
@ -658,7 +658,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
}
if err := query.SetDownloadFileID(downloadedAs, file.ID); err != nil {
log.Errorf("index: %s for %s", err, logName)
log.Errorf("index: %s in %s (set download id)", err, logName)
}
// Write YAML sidecar file (optional).
@ -666,7 +666,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
yamlFile := photo.YamlFileName(Config().OriginalsPath(), Config().SidecarPath())
if err := photo.SaveAsYaml(yamlFile); err != nil {
log.Errorf("index: %s (update yaml) for %s", err.Error(), logName)
log.Errorf("index: %s in %s (update yaml)", err.Error(), logName)
} else {
log.Infof("index: updated yaml file %s", txt.Quote(fs.RelName(yamlFile, Config().OriginalsPath())))
}

2
internal/photoprism/mediafile.go
View file

@ -862,7 +862,7 @@ func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
img, err := imaging.Open(m.FileName(), imaging.AutoOrientation(true))
if err != nil {
log.Errorf("mediafile: can't open %s (%s)", txt.Quote(m.FileName()), err.Error())
log.Errorf("mediafile: %s in %s", err.Error(), txt.Quote(m.BaseName()))
return err
}

6
internal/query/errors.go
View file

@ -12,7 +12,11 @@ func Errors(limit, offset int, s string) (results entity.Errors, err error) {
s = strings.TrimSpace(s)
if len(s) >= 3 {
if s == "errors" {
stmt = stmt.Where("error_level = 'error'")
} else if s == "warnings" {
stmt = stmt.Where("error_level = 'warning'")
} else if len(s) >= 3 {
stmt = stmt.Where("error_message LIKE ?", "%"+s+"%")
}