From 8cfabe32051579756f08f65994d08e33ad3528fc Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Wed, 30 Dec 2020 13:33:47 +0100 Subject: [PATCH] Metadata: Cache ExifTool JSON by original file hash #755 #759 --- internal/commands/reset.go | 77 ++++++++--- internal/meta/json.go | 7 + internal/photoprism/convert.go | 130 ++++++++++-------- internal/photoprism/convert_test.go | 56 +++----- internal/photoprism/filename.go | 26 ++++ internal/photoprism/import_worker.go | 16 +-- internal/photoprism/index_related.go | 17 +-- internal/photoprism/mediafile.go | 25 ---- ...etadata_test.go => mediafile_meta_test.go} | 72 ++++++++++ internal/photoprism/mediafile_test.go | 74 +--------- pkg/fs/cache.go | 28 ++++ pkg/fs/cache_test.go | 44 ++++++ 12 files changed, 337 insertions(+), 235 deletions(-) rename internal/photoprism/{metadata_test.go => mediafile_meta_test.go} (84%) create mode 100644 pkg/fs/cache.go create mode 100644 pkg/fs/cache_test.go diff --git a/internal/commands/reset.go b/internal/commands/reset.go index fbe73ae4e..49a8188e8 100644 --- a/internal/commands/reset.go +++ b/internal/commands/reset.go @@ -14,30 +14,22 @@ import ( "github.com/urfave/cli" ) -// ResetCommand resets the index and optionally removes YAML sidecar backup files. +// ResetCommand resets the index and removes sidecar files after confirmation. var ResetCommand = cli.Command{ Name: "reset", - Usage: "Resets the index and optionally removes YAML sidecar backup files", + Usage: "Resets the index and removes sidecar files after confirmation", Action: resetAction, } -// resetAction removes the index and sidecar files after asking for confirmation. +// resetAction resets the index and removes sidecar files after confirmation. func resetAction(ctx *cli.Context) error { - log.Warnf("'photoprism reset' removes ALL data incl albums from the existing database") + log.Warnf("'photoprism reset' resets the index and removes sidecar files after confirmation") removeIndex := promptui.Prompt{ - Label: "Reset index database?", + Label: "Reset index database incl albums, labels, users and metadata?", IsConfirm: true, } - _, err := removeIndex.Run() - - if err != nil { - return fmt.Errorf("abort") - } - - start := time.Now() - conf := config.NewConfig(ctx) _, cancel := context.WithCancel(context.Background()) defer cancel() @@ -48,20 +40,61 @@ func resetAction(ctx *cli.Context) error { entity.SetDbProvider(conf) - tables := entity.Entities + if _, err := removeIndex.Run(); err == nil { + start := time.Now() - log.Infoln("dropping existing tables") - tables.Drop() + tables := entity.Entities - log.Infoln("restoring default schema") - entity.MigrateDb() + log.Infoln("dropping existing tables") + tables.Drop() - if conf.AdminPassword() != "" { - log.Infoln("restoring initial admin password") - entity.Admin.InitPassword(conf.AdminPassword()) + log.Infoln("restoring default schema") + entity.MigrateDb() + + if conf.AdminPassword() != "" { + log.Infoln("restoring initial admin password") + entity.Admin.InitPassword(conf.AdminPassword()) + } + + log.Infof("database reset completed in %s", time.Since(start)) + } else { + log.Infof("keeping index database") } - log.Infof("database reset completed in %s", time.Since(start)) + removeSidecarJson := promptui.Prompt{ + Label: "Permanently delete all *.json photo sidecar files?", + IsConfirm: true, + } + + if _, err := removeSidecarJson.Run(); err == nil { + start := time.Now() + + matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json") + + if err != nil { + return err + } + + if len(matches) > 0 { + log.Infof("%d json photo sidecar files will be removed", len(matches)) + + for _, name := range matches { + if err := os.Remove(name); err != nil { + fmt.Print("E") + } else { + fmt.Print(".") + } + } + + fmt.Println("") + + log.Infof("removed json files in %s", time.Since(start)) + } else { + log.Infof("no json files found") + } + } else { + log.Infof("keeping json sidecar files") + } removeSidecarYaml := promptui.Prompt{ Label: "Permanently delete all *.yml photo metadata backups?", diff --git a/internal/meta/json.go b/internal/meta/json.go index 6c3ec9ba3..489e5b68b 100644 --- a/internal/meta/json.go +++ b/internal/meta/json.go @@ -7,6 +7,8 @@ import ( "path/filepath" "runtime/debug" + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/txt" ) @@ -30,6 +32,11 @@ func (data *Data) JSON(jsonName, originalName string) (err error) { } quotedName := txt.Quote(filepath.Base(jsonName)) + + if !fs.FileExists(jsonName) { + return fmt.Errorf("json file %s not found", quotedName) + } + jsonData, err := ioutil.ReadFile(jsonName) if err != nil { diff --git a/internal/photoprism/convert.go b/internal/photoprism/convert.go index aae6e5c3f..18a646063 100644 --- a/internal/photoprism/convert.go +++ b/internal/photoprism/convert.go @@ -88,16 +88,16 @@ func (c *Convert) Start(path string) error { return result } - mf, err := NewMediaFile(fileName) + f, err := NewMediaFile(fileName) - if err != nil || !(mf.IsRaw() || mf.IsHEIF() || mf.IsImageOther()) { + if err != nil || !(f.IsRaw() || f.IsHEIF() || f.IsImageOther()) { return nil } done[fileName] = fs.Processed jobs <- ConvertJob{ - image: mf, + image: f, convert: c, } @@ -114,26 +114,26 @@ func (c *Convert) Start(path string) error { } // ToJson uses exiftool to export metadata to a json file. -func (c *Convert) ToJson(mf *MediaFile) (*MediaFile, error) { - jsonName := fs.FormatJson.FindFirst(mf.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) - - result, err := NewMediaFile(jsonName) - - if err == nil { - return result, nil +func (c *Convert) ToJson(f *MediaFile) (jsonName string, err error) { + if f == nil { + return "", fmt.Errorf("exiftool: file is nil (found a bug?)") } - if !c.conf.SidecarWritable() { - return nil, fmt.Errorf("convert: can't create json sidecar file for %s in read only mode", txt.Quote(mf.BaseName())) + jsonName, err = f.ExifToolJsonName() + + if err != nil { + return "", nil } - jsonName = fs.FileName(mf.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), ".json") + if fs.FileExists(jsonName) { + return jsonName, nil + } - fileName := mf.RelName(c.conf.OriginalsPath()) + relName := f.RelName(c.conf.OriginalsPath()) - log.Debugf("convert: %s -> %s", fileName, filepath.Base(jsonName)) + log.Infof("exiftool: extracting metadata from %s", relName) - cmd := exec.Command(c.conf.ExifToolBin(), "-j", mf.FileName()) + cmd := exec.Command(c.conf.ExifToolBin(), "-j", f.FileName()) // Fetch command output. var out bytes.Buffer @@ -144,42 +144,42 @@ func (c *Convert) ToJson(mf *MediaFile) (*MediaFile, error) { // Run convert command. if err := cmd.Run(); err != nil { if stderr.String() != "" { - return nil, errors.New(stderr.String()) + return "", errors.New(stderr.String()) } else { - return nil, err + return "", err } } // Write output to file. if err := ioutil.WriteFile(jsonName, []byte(out.String()), os.ModePerm); err != nil { - return nil, err + return "", err } // Check if file exists. if !fs.FileExists(jsonName) { - return nil, fmt.Errorf("convert: %s could not be created, check configuration", jsonName) + return "", fmt.Errorf("exiftool: failed creating %s", filepath.Base(jsonName)) } - return NewMediaFile(jsonName) + return jsonName, err } // JpegConvertCommand returns the command for converting files to JPEG, depending on the format. -func (c *Convert) JpegConvertCommand(mf *MediaFile, jpegName string, xmpName string) (result *exec.Cmd, useMutex bool, err error) { +func (c *Convert) JpegConvertCommand(f *MediaFile, jpegName string, xmpName string) (result *exec.Cmd, useMutex bool, err error) { size := strconv.Itoa(c.conf.JpegSize()) - if mf.IsRaw() { + if f.IsRaw() { if c.conf.SipsBin() != "" { - result = exec.Command(c.conf.SipsBin(), "-Z", size, "-s", "format", "jpeg", "--out", jpegName, mf.FileName()) - } else if c.conf.DarktableBin() != "" && mf.Extension() != ".cr3" { + result = exec.Command(c.conf.SipsBin(), "-Z", size, "-s", "format", "jpeg", "--out", jpegName, f.FileName()) + } else if c.conf.DarktableBin() != "" && f.Extension() != ".cr3" { var args []string // Only one instance of darktable-cli allowed due to locking if presets are loaded. if c.conf.DarktablePresets() { useMutex = true - args = []string{"--width", size, "--height", size, mf.FileName()} + args = []string{"--width", size, "--height", size, f.FileName()} } else { useMutex = false - args = []string{"--apply-custom-presets", "false", "--width", size, "--height", size, mf.FileName()} + args = []string{"--apply-custom-presets", "false", "--width", size, "--height", size, f.FileName()} } if xmpName != "" { @@ -193,34 +193,38 @@ func (c *Convert) JpegConvertCommand(mf *MediaFile, jpegName string, xmpName str jpegQuality := fmt.Sprintf("-j%d", c.conf.JpegQuality()) profile := filepath.Join(conf.AssetsPath(), "profiles", "raw.pp3") - args := []string{"-o", jpegName, "-p", profile, "-d", jpegQuality, "-js3", "-b8", "-c", mf.FileName()} + args := []string{"-o", jpegName, "-p", profile, "-d", jpegQuality, "-js3", "-b8", "-c", f.FileName()} result = exec.Command(c.conf.RawtherapeeBin(), args...) } else { - return nil, useMutex, fmt.Errorf("convert: no converter found for %s", txt.Quote(mf.BaseName())) + return nil, useMutex, fmt.Errorf("convert: no converter found for %s", txt.Quote(f.BaseName())) } - } else if mf.IsVideo() { - result = exec.Command(c.conf.FFmpegBin(), "-y", "-i", mf.FileName(), "-ss", "00:00:00.001", "-vframes", "1", jpegName) - } else if mf.IsHEIF() { - result = exec.Command(c.conf.HeifConvertBin(), mf.FileName(), jpegName) + } else if f.IsVideo() { + result = exec.Command(c.conf.FFmpegBin(), "-y", "-i", f.FileName(), "-ss", "00:00:00.001", "-vframes", "1", jpegName) + } else if f.IsHEIF() { + result = exec.Command(c.conf.HeifConvertBin(), f.FileName(), jpegName) } else { - return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", mf.FileType(), txt.Quote(mf.BaseName())) + return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", f.FileType(), txt.Quote(f.BaseName())) } return result, useMutex, nil } // ToJpeg converts a single image file to JPEG if possible. -func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) { - if !image.Exists() { - return nil, fmt.Errorf("convert: can not convert to jpeg, file does not exist (%s)", image.RelName(c.conf.OriginalsPath())) +func (c *Convert) ToJpeg(f *MediaFile) (*MediaFile, error) { + if f == nil { + return nil, fmt.Errorf("convert: file is nil (found a bug?)") } - if image.IsJpeg() { - return image, nil + if !f.Exists() { + return nil, fmt.Errorf("convert: can not convert to jpeg, file does not exist (%s)", f.RelName(c.conf.OriginalsPath())) } - jpegName := fs.FormatJpeg.FindFirst(image.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) + if f.IsJpeg() { + return f, nil + } + + jpegName := fs.FormatJpeg.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) mediaFile, err := NewMediaFile(jpegName) @@ -229,25 +233,25 @@ func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) { } if !c.conf.SidecarWritable() { - return nil, fmt.Errorf("convert: disabled in read only mode (%s)", image.RelName(c.conf.OriginalsPath())) + return nil, fmt.Errorf("convert: disabled in read only mode (%s)", f.RelName(c.conf.OriginalsPath())) } - jpegName = fs.FileName(image.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.JpegExt) - fileName := image.RelName(c.conf.OriginalsPath()) + jpegName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.JpegExt) + fileName := f.RelName(c.conf.OriginalsPath()) log.Debugf("convert: %s -> %s", fileName, filepath.Base(jpegName)) - xmpName := fs.FormatXMP.Find(image.FileName(), false) + xmpName := fs.FormatXMP.Find(f.FileName(), false) event.Publish("index.converting", event.Data{ - "fileType": image.FileType(), + "fileType": f.FileType(), "fileName": fileName, "baseName": filepath.Base(fileName), "xmpName": filepath.Base(xmpName), }) - if image.IsImageOther() { - _, err = thumb.Jpeg(image.FileName(), jpegName) + if f.IsImageOther() { + _, err = thumb.Jpeg(f.FileName(), jpegName) if err != nil { return nil, err @@ -256,7 +260,7 @@ func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) { return NewMediaFile(jpegName) } - cmd, useMutex, err := c.JpegConvertCommand(image, jpegName, xmpName) + cmd, useMutex, err := c.JpegConvertCommand(f, jpegName, xmpName) if err != nil { return nil, err @@ -292,31 +296,35 @@ func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) { } // AvcConvertCommand returns the command for converting video files to MPEG-4 AVC. -func (c *Convert) AvcConvertCommand(mf *MediaFile, avcName string) (result *exec.Cmd, useMutex bool, err error) { - if mf.IsVideo() { +func (c *Convert) AvcConvertCommand(f *MediaFile, avcName string) (result *exec.Cmd, useMutex bool, err error) { + if f.IsVideo() { // Don't transcode more than one video at the same time. useMutex = true result = exec.Command( c.conf.FFmpegBin(), - "-i", mf.FileName(), + "-i", f.FileName(), "-c:v", "libx264", "-f", "mp4", avcName, ) } else { - return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", mf.FileType(), txt.Quote(mf.BaseName())) + return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", f.FileType(), txt.Quote(f.BaseName())) } return result, useMutex, nil } // ToAvc converts a single video file to MPEG-4 AVC. -func (c *Convert) ToAvc(video *MediaFile) (*MediaFile, error) { - if !video.Exists() { - return nil, fmt.Errorf("convert: can not convert to avc1, file does not exist (%s)", video.RelName(c.conf.OriginalsPath())) +func (c *Convert) ToAvc(f *MediaFile) (*MediaFile, error) { + if f == nil { + return nil, fmt.Errorf("convert: file is nil (found a bug?)") } - avcName := fs.FormatAvc.FindFirst(video.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) + if !f.Exists() { + return nil, fmt.Errorf("convert: can not convert to avc1, file does not exist (%s)", f.RelName(c.conf.OriginalsPath())) + } + + avcName := fs.FormatAvc.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) mediaFile, err := NewMediaFile(avcName) @@ -325,22 +333,22 @@ func (c *Convert) ToAvc(video *MediaFile) (*MediaFile, error) { } if !c.conf.SidecarWritable() { - return nil, fmt.Errorf("convert: disabled in read only mode (%s)", video.RelName(c.conf.OriginalsPath())) + return nil, fmt.Errorf("convert: disabled in read only mode (%s)", f.RelName(c.conf.OriginalsPath())) } - avcName = fs.FileName(video.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.AvcExt) - fileName := video.RelName(c.conf.OriginalsPath()) + avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.AvcExt) + fileName := f.RelName(c.conf.OriginalsPath()) log.Debugf("convert: %s -> %s", fileName, filepath.Base(avcName)) event.Publish("index.converting", event.Data{ - "fileType": video.FileType(), + "fileType": f.FileType(), "fileName": fileName, "baseName": filepath.Base(fileName), "xmpName": "", }) - cmd, useMutex, err := c.AvcConvertCommand(video, avcName) + cmd, useMutex, err := c.AvcConvertCommand(f, avcName) if err != nil { log.Error(err) diff --git a/internal/photoprism/convert_test.go b/internal/photoprism/convert_test.go index 169e2d7da..c5f2398fd 100644 --- a/internal/photoprism/convert_test.go +++ b/internal/photoprism/convert_test.go @@ -119,12 +119,8 @@ func TestConvert_ToJson(t *testing.T) { t.Run("gopher-video.mp4", func(t *testing.T) { fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4") - outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.mp4.json") - - _ = os.Remove(outputName) assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName) - assert.Falsef(t, fs.FileExists(outputName), "output file must not exist: %s", outputName) mf, err := NewMediaFile(fileName) @@ -132,34 +128,24 @@ func TestConvert_ToJson(t *testing.T) { t.Fatal(err) } - jsonFile, err := convert.ToJson(mf) + jsonName, err := convert.ToJson(mf) if err != nil { t.Fatal(err) } - if jsonFile == nil { - t.Fatal("jsonFile should not be nil") + if jsonName == "" { + t.Fatal("json file name should not be empty") } - assert.Equal(t, jsonFile.FileName(), outputName) - assert.Truef(t, fs.FileExists(jsonFile.FileName()), "output file does not exist: %s", jsonFile.FileName()) - assert.False(t, jsonFile.IsJpeg()) - assert.False(t, jsonFile.IsMedia()) - assert.False(t, jsonFile.IsVideo()) - assert.True(t, jsonFile.IsSidecar()) + assert.FileExists(t, jsonName) - _ = os.Remove(outputName) + _ = os.Remove(jsonName) }) t.Run("IMG_4120.JPG", func(t *testing.T) { fileName := filepath.Join(conf.ExamplesPath(), "IMG_4120.JPG") - outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "IMG_4120.JPG.json") - - _ = os.Remove(outputName) - assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName) - assert.Falsef(t, fs.FileExists(outputName), "output file must not exist: %s", outputName) mf, err := NewMediaFile(fileName) @@ -167,32 +153,25 @@ func TestConvert_ToJson(t *testing.T) { t.Fatal(err) } - jsonFile, err := convert.ToJson(mf) + jsonName, err := convert.ToJson(mf) if err != nil { t.Fatal(err) } - if jsonFile == nil { - t.Fatal("jsonFile should not be nil") + if jsonName == "" { + t.Fatal("json file name should not be empty") } - assert.Equal(t, jsonFile.FileName(), outputName) - assert.Truef(t, fs.FileExists(jsonFile.FileName()), "output file does not exist: %s", jsonFile.FileName()) - assert.False(t, jsonFile.IsJpeg()) - assert.False(t, jsonFile.IsMedia()) - assert.False(t, jsonFile.IsVideo()) - assert.True(t, jsonFile.IsSidecar()) + assert.FileExists(t, jsonName) - _ = os.Remove(outputName) + _ = os.Remove(jsonName) }) t.Run("iphone_7.heic", func(t *testing.T) { fileName := conf.ExamplesPath() + "/iphone_7.heic" - outputName := conf.ExamplesPath() + "/iphone_7.json" assert.True(t, fs.FileExists(fileName)) - assert.True(t, fs.FileExists(outputName)) mf, err := NewMediaFile(fileName) @@ -200,18 +179,19 @@ func TestConvert_ToJson(t *testing.T) { t.Fatal(err) } - jsonFile, err := convert.ToJson(mf) + jsonName, err := convert.ToJson(mf) if err != nil { t.Fatal(err) } - assert.Equal(t, jsonFile.FileName(), outputName) - assert.Truef(t, fs.FileExists(jsonFile.FileName()), "output file does not exist: %s", jsonFile.FileName()) - assert.False(t, jsonFile.IsJpeg()) - assert.False(t, jsonFile.IsMedia()) - assert.False(t, jsonFile.IsVideo()) - assert.True(t, jsonFile.IsSidecar()) + if jsonName == "" { + t.Fatal("json file name should not be empty") + } + + assert.FileExists(t, jsonName) + + _ = os.Remove(jsonName) }) } diff --git a/internal/photoprism/filename.go b/internal/photoprism/filename.go index 89a65ace0..e0d392a05 100644 --- a/internal/photoprism/filename.go +++ b/internal/photoprism/filename.go @@ -1,7 +1,11 @@ package photoprism import ( + "fmt" "path" + "path/filepath" + + "github.com/photoprism/photoprism/pkg/fs" "github.com/photoprism/photoprism/internal/entity" ) @@ -18,3 +22,25 @@ func FileName(fileRoot, fileName string) string { return path.Join(Config().OriginalsPath(), fileName) } } + +// CachePath returns a cache directory name based on the base path, file hash and cache namespace. +func CachePath(fileHash, namespace string) (cachePath string, err error) { + return fs.CachePath(Config().CachePath(), fileHash, namespace, true) +} + +// CacheName returns an absolute cache file name based on the base path, file hash and cache namespace. +func CacheName(fileHash, namespace, cacheKey string) (cacheName string, err error) { + if cacheKey == "" { + return "", fmt.Errorf("cache: key for hash '%s' is empty", fileHash) + } + + cachePath, err := CachePath(fileHash, namespace) + + if err != nil { + return "", err + } + + cacheName = filepath.Join(cachePath, fmt.Sprintf("%s_%s", fileHash, cacheKey)) + + return cacheName, nil +} diff --git a/internal/photoprism/import_worker.go b/internal/photoprism/import_worker.go index 75ad681ba..607bb9c85 100644 --- a/internal/photoprism/import_worker.go +++ b/internal/photoprism/import_worker.go @@ -105,11 +105,11 @@ func ImportWorker(jobs <-chan ImportJob) { } } - if f.NeedsJson() { - if jsonFile, err := imp.convert.ToJson(f); err != nil { - log.Errorf("import: %s in %s (create json sidecar)", err.Error(), txt.Quote(f.BaseName())) + if f.NeedsExifToolJson() { + if jsonName, err := imp.convert.ToJson(f); err != nil { + log.Errorf("import: %s in %s (extract json metadata)", err.Error(), txt.Quote(f.BaseName())) } else { - log.Debugf("import: %s created", txt.Quote(jsonFile.BaseName())) + log.Debugf("import: %s created", filepath.Base(jsonName)) } } @@ -167,11 +167,11 @@ func ImportWorker(jobs <-chan ImportJob) { continue } - if f.NeedsJson() { - if jsonFile, err := ind.convert.ToJson(f); err != nil { - log.Errorf("import: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error()) + if f.NeedsExifToolJson() { + if jsonName, err := imp.convert.ToJson(f); err != nil { + log.Errorf("import: %s in %s (extract json metadata)", err.Error(), txt.Quote(f.BaseName())) } else { - log.Debugf("import: %s created", txt.Quote(jsonFile.BaseName())) + log.Debugf("import: %s created", filepath.Base(jsonName)) } } diff --git a/internal/photoprism/index_related.go b/internal/photoprism/index_related.go index 28f6453bd..263cb236c 100644 --- a/internal/photoprism/index_related.go +++ b/internal/photoprism/index_related.go @@ -2,6 +2,7 @@ package photoprism import ( "fmt" + "path/filepath" "github.com/photoprism/photoprism/internal/query" "github.com/photoprism/photoprism/pkg/txt" @@ -46,11 +47,11 @@ func IndexMain(related *RelatedFiles, ind *Index, opt IndexOptions) (result Inde } } - if f.NeedsJson() { - if jsonFile, err := ind.convert.ToJson(f); err != nil { - log.Errorf("index: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error()) + if f.NeedsExifToolJson() { + if jsonName, err := ind.convert.ToJson(f); err != nil { + log.Errorf("index: %s in %s (extract json metadata)", err.Error(), txt.Quote(f.BaseName())) } else { - log.Debugf("index: %s created", txt.Quote(jsonFile.BaseName())) + log.Debugf("index: %s created", filepath.Base(jsonName)) } } @@ -102,11 +103,11 @@ func IndexRelated(related RelatedFiles, ind *Index, opt IndexOptions) (result In continue } - if f.NeedsJson() { - if jsonFile, err := ind.convert.ToJson(f); err != nil { - log.Errorf("index: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error()) + if f.NeedsExifToolJson() { + if jsonName, err := ind.convert.ToJson(f); err != nil { + log.Errorf("index: %s in %s (extract json metadata)", err.Error(), txt.Quote(f.BaseName())) } else { - log.Debugf("index: %s created", txt.Quote(jsonFile.BaseName())) + log.Debugf("index: %s created", filepath.Base(jsonName)) } } diff --git a/internal/photoprism/mediafile.go b/internal/photoprism/mediafile.go index 5b3b4d719..8df02b15b 100644 --- a/internal/photoprism/mediafile.go +++ b/internal/photoprism/mediafile.go @@ -270,17 +270,6 @@ func (m *MediaFile) EditedName() string { return "" } -// JsonName returns the corresponding JSON sidecar file name as used by Google Photos (and potentially other apps). -func (m *MediaFile) JsonName() string { - jsonName := m.fileName + ".json" - - if fs.FileExists(jsonName) { - return jsonName - } - - return "" -} - // RelatedFiles returns files which are related to this file. func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err error) { var prefix string @@ -795,20 +784,6 @@ func (m *MediaFile) HasJpeg() bool { return m.hasJpeg } -// HasJson returns true if this file has or is a json sidecar file. -func (m *MediaFile) HasJson() bool { - if m.IsJson() { - return true - } - - return fs.FormatJson.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false) != "" -} - -// NeedsJson tests if the media file needs a JSON sidecar file to be created. -func (m *MediaFile) NeedsJson() bool { - return Config().ExifToolJson() && m.Root() != entity.RootSidecar && m.IsMedia() && !m.HasJson() -} - func (m *MediaFile) decodeDimensions() error { if !m.IsMedia() { return fmt.Errorf("failed decoding dimensions for %s", txt.Quote(m.BaseName())) diff --git a/internal/photoprism/metadata_test.go b/internal/photoprism/mediafile_meta_test.go similarity index 84% rename from internal/photoprism/metadata_test.go rename to internal/photoprism/mediafile_meta_test.go index 1d37ea9cc..763e4507e 100644 --- a/internal/photoprism/metadata_test.go +++ b/internal/photoprism/mediafile_meta_test.go @@ -10,6 +10,78 @@ import ( "github.com/stretchr/testify/assert" ) +func TestMediaFile_HasSidecarJson(t *testing.T) { + t.Run("false", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg") + + if err != nil { + t.Fatal(err) + } + + assert.False(t, mediaFile.HasSidecarJson()) + }) + t.Run("true", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4") + + if err != nil { + t.Fatal(err) + } + + assert.True(t, mediaFile.HasSidecarJson()) + }) + t.Run("true", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4.json") + + if err != nil { + t.Fatal(err) + } + + assert.True(t, mediaFile.HasSidecarJson()) + }) +} + +func TestMediaFile_NeedsExifToolJson(t *testing.T) { + t.Run("false", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg") + + if err != nil { + t.Fatal(err) + } + + assert.True(t, mediaFile.NeedsExifToolJson()) + }) + t.Run("true", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4") + + if err != nil { + t.Fatal(err) + } + + assert.True(t, mediaFile.NeedsExifToolJson()) + }) + t.Run("true", func(t *testing.T) { + conf := config.TestConfig() + + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4.json") + + if err != nil { + t.Fatal(err) + } + + assert.False(t, mediaFile.NeedsExifToolJson()) + }) +} + func TestMediaFile_Exif_JPEG(t *testing.T) { conf := config.TestConfig() diff --git a/internal/photoprism/mediafile_test.go b/internal/photoprism/mediafile_test.go index c8a7a51ca..fbb8f3779 100644 --- a/internal/photoprism/mediafile_test.go +++ b/internal/photoprism/mediafile_test.go @@ -1887,7 +1887,7 @@ func TestMediaFile_JsonName(t *testing.T) { t.Fatal(err) } - name := mediaFile.JsonName() + name := mediaFile.SidecarJsonName() assert.True(t, strings.HasSuffix(name, "/assets/examples/blue-go-video.mp4.json")) }) } @@ -2061,78 +2061,6 @@ func TestMediaFile_IsPlayableVideo(t *testing.T) { }) } -func TestMediaFile_HasJson(t *testing.T) { - t.Run("false", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg") - - if err != nil { - t.Fatal(err) - } - - assert.False(t, mediaFile.HasJson()) - }) - t.Run("true", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4") - - if err != nil { - t.Fatal(err) - } - - assert.True(t, mediaFile.HasJson()) - }) - t.Run("true", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4.json") - - if err != nil { - t.Fatal(err) - } - - assert.True(t, mediaFile.HasJson()) - }) -} - -func TestMediaFile_NeedsJson(t *testing.T) { - t.Run("false", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg") - - if err != nil { - t.Fatal(err) - } - - assert.True(t, mediaFile.NeedsJson()) - }) - t.Run("true", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4") - - if err != nil { - t.Fatal(err) - } - - assert.False(t, mediaFile.NeedsJson()) - }) - t.Run("true", func(t *testing.T) { - conf := config.TestConfig() - - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4.json") - - if err != nil { - t.Fatal(err) - } - - assert.False(t, mediaFile.NeedsJson()) - }) -} - func TestMediaFile_RenameSidecars(t *testing.T) { t.Run("success", func(t *testing.T) { conf := config.TestConfig() diff --git a/pkg/fs/cache.go b/pkg/fs/cache.go new file mode 100644 index 000000000..fdf970b67 --- /dev/null +++ b/pkg/fs/cache.go @@ -0,0 +1,28 @@ +package fs + +import ( + "fmt" + "os" + "path" +) + +// CachePath returns a cache directory name based on the base path, file hash and cache namespace. +func CachePath(basePath, fileHash, namespace string, create bool) (cachePath string, err error) { + if len(fileHash) < 4 { + return "", fmt.Errorf("cache: hash '%s' is too short", fileHash) + } + + if namespace == "" { + return "", fmt.Errorf("cache: namespace for hash '%s' is empty", fileHash) + } + + cachePath = path.Join(basePath, namespace, fileHash[0:1], fileHash[1:2], fileHash[2:3]) + + if create { + if err := os.MkdirAll(cachePath, os.ModePerm); err != nil { + return "", err + } + } + + return cachePath, nil +} diff --git a/pkg/fs/cache_test.go b/pkg/fs/cache_test.go new file mode 100644 index 000000000..f4b87f7c2 --- /dev/null +++ b/pkg/fs/cache_test.go @@ -0,0 +1,44 @@ +package fs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCachePath(t *testing.T) { + t.Run("error", func(t *testing.T) { + result, err := CachePath("/foo/bar", "123", "baz", false) + + assert.Equal(t, "", result) + assert.EqualError(t, err, "cache: hash '123' is too short") + }) + + t.Run("1234567890abcdef", func(t *testing.T) { + result, err := CachePath("/foo/bar", "1234567890abcdef", "baz", false) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "/foo/bar/baz/1/2/3", result) + }) + + t.Run("create", func(t *testing.T) { + ns := "pkg_fs_test" + result, err := CachePath(os.TempDir(), "1234567890abcdef", ns, true) + + if err != nil { + t.Fatal(err) + } + + expected := filepath.Join(os.TempDir(), ns, "1", "2", "3") + + assert.Equal(t, expected, result) + assert.DirExists(t, expected) + + _ = os.Remove(expected) + }) +}