Library: Stack sidecar files with vendor specific naming schemes #2983

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/photoprism/mediafile_related.go

@@ -9,18 +9,13 @@ import (
 
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
+	"github.com/photoprism/photoprism/pkg/list"
 )
 
-// RelatedFilePathPrefix returns the absolute file path and name prefix without file extensions
-// and suffixes to be ignored.
-func (m *MediaFile) RelatedFilePathPrefix(stripSequence bool) (s string) {
-	return fs.RelatedFilePathPrefix(m.FileName(), stripSequence)
-}
-
 // RelatedFiles returns files which are related to this file.
 func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err error) {
 	// Related file path prefix without ignored file name extensions and suffixes.
-	filePathPrefix := m.RelatedFilePathPrefix(stripSequence)
+	filePathPrefix := m.AbsPrefix(stripSequence)
 
 	// Storage folder path prefixes.
 	sidecarPrefix := Config().SidecarPath() + "/"
@@ -58,9 +53,10 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e
 		return result, err
 	}
 
-	// Additionally include edited version in the file matches, if exists.
-	if name := m.EditedName(); name != "" {
-		matches = append(matches, name)
+	// Find additional sidecar files with naming schemes not matching the glob pattern,
+	// see https://github.com/photoprism/photoprism/issues/2983 for further information.
+	if files, _ := m.RelatedSidecarFiles(stripSequence); len(files) > 0 {
+		matches = list.Join(matches, files)
 	}
 
 	isHEIC := false
@@ -138,3 +134,33 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e
 
 	return result, nil
 }
+
+// RelatedSidecarFiles finds additional sidecar files with naming schemes not matching the default glob pattern
+// for related files. see https://github.com/photoprism/photoprism/issues/2983 for further information.
+func (m *MediaFile) RelatedSidecarFiles(stripSequence bool) (files []string, err error) {
+	baseName := filepath.Base(m.fileName)
+	files = make([]string, 0, 2)
+
+	// Find edited file versions with a naming scheme as used by Apple, for example "IMG_E12345.JPG".
+	if strings.ToUpper(baseName[:4]) == "IMG_" && strings.ToUpper(baseName[:5]) != "IMG_E" {
+		if fileName := filepath.Join(filepath.Dir(m.fileName), baseName[:4]+"E"+baseName[4:]); fs.FileExists(fileName) {
+			files = append(files, fileName)
+		}
+	}
+
+	// Related file path prefix without ignored file name extensions and suffixes.
+	filePathPrefix := m.AbsPrefix(stripSequence)
+
+	// Find additional sidecar files that match the default glob pattern for related files.
+	globPattern := regexp.QuoteMeta(filePathPrefix) + "_????\\.*"
+	matches, err := filepath.Glob(globPattern)
+
+	if err != nil {
+		return files, err
+	}
+
+	// Add glob file matches to results.
+	files = append(files, matches...)
+
+	return files, nil
+}

internal/photoprism/mediafile_related_test.go

@@ -8,35 +8,6 @@ import (
 	"github.com/photoprism/photoprism/internal/config"
 )
 
-func TestMediaFile_RelatedFilePathPrefix(t *testing.T) {
-	t.Run("IMG_1234_HEVC.JPEG", func(t *testing.T) {
-		fileName := "testdata/related/IMG_1234_HEVC (3).JPEG"
-		f, err := NewMediaFile(fileName)
-
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		assert.Equal(t, fileName, f.FileName())
-		assert.Equal(t, "testdata/related/IMG_1234_HEVC", f.AbsPrefix(true))
-		assert.Equal(t, "testdata/related/IMG_1234_HEVC (3)", f.AbsPrefix(false))
-		assert.Equal(t, "testdata/related/IMG_1234", f.RelatedFilePathPrefix(true))
-		assert.Equal(t, "testdata/related/IMG_1234_HEVC (3)", f.RelatedFilePathPrefix(false))
-	})
-	t.Run("fern_green.jpg", func(t *testing.T) {
-		f, err := NewMediaFile(conf.ExamplesPath() + "/fern_green.jpg")
-
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		expected := conf.ExamplesPath() + "/fern_green"
-
-		assert.Equal(t, expected, f.RelatedFilePathPrefix(true))
-		assert.Equal(t, expected, f.RelatedFilePathPrefix(false))
-	})
-}
-
 func TestMediaFile_RelatedFiles(t *testing.T) {
 	c := config.TestConfig()
 
@@ -227,30 +198,67 @@ func TestMediaFile_RelatedFiles(t *testing.T) {
 		assert.Equal(t, "2015-02-04.jpg.json", related.Files[2].BaseName())
 		assert.Equal(t, "2015-02-04.jpg(1).json", related.Files[3].BaseName())
 	})
+
+	t.Run("Ordering", func(t *testing.T) {
+		mediaFile, err := NewMediaFile(c.ExamplesPath() + "/IMG_4120.JPG")
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		related, err := mediaFile.RelatedFiles(true)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Len(t, related.Files, 5)
+
+		assert.Equal(t, c.ExamplesPath()+"/IMG_4120.AAE", related.Files[0].FileName())
+		assert.Equal(t, c.ExamplesPath()+"/IMG_4120.JPG", related.Files[1].FileName())
+
+		for _, result := range related.Files {
+			filename := result.FileName()
+			t.Logf("FileName: %s", filename)
+		}
+	})
 }
 
-func TestMediaFile_RelatedFiles_Ordering(t *testing.T) {
-	c := config.TestConfig()
+func TestMediaFile_RelatedSidecarFiles(t *testing.T) {
+	t.Run("FindEdited", func(t *testing.T) {
+		file, err := NewMediaFile("testdata/related/IMG_1234 (2).JPEG")
 
-	mediaFile, err := NewMediaFile(c.ExamplesPath() + "/IMG_4120.JPG")
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	if err != nil {
-		t.Fatal(err)
-	}
+		files, err := file.RelatedSidecarFiles(false)
 
-	related, err := mediaFile.RelatedFiles(true)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	if err != nil {
-		t.Fatal(err)
-	}
+		expected := []string{"testdata/related/IMG_E1234 (2).JPEG"}
 
-	assert.Len(t, related.Files, 5)
+		assert.Len(t, files, len(expected))
+		assert.Equal(t, expected, files)
+	})
+	t.Run("StripSequence", func(t *testing.T) {
+		file, err := NewMediaFile("testdata/related/IMG_1234 (2).JPEG")
 
-	assert.Equal(t, c.ExamplesPath()+"/IMG_4120.AAE", related.Files[0].FileName())
-	assert.Equal(t, c.ExamplesPath()+"/IMG_4120.JPG", related.Files[1].FileName())
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	for _, result := range related.Files {
-		filename := result.FileName()
-		t.Logf("FileName: %s", filename)
-	}
+		files, err := file.RelatedSidecarFiles(true)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expected := []string{"testdata/related/IMG_E1234 (2).JPEG", "testdata/related/IMG_1234_HEVC.JPEG"}
+
+		assert.Len(t, files, len(expected))
+		assert.Equal(t, expected, files)
+	})
 }

pkg/fs/filepath.go

@@ -2,15 +2,10 @@ package fs
 
 import (
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 )
 
-// RelatedMediaFileSuffix is a regular expression that matches suffixes of related media files,
-// see https://github.com/photoprism/photoprism/issues/2983 (Support Live Photos downloaded with "iCloudPD").
-var RelatedMediaFileSuffix = regexp.MustCompile(`(?i)_(jpg|jpeg|hevc)$`)
-
 // StripSequence removes common sequence patterns at the end of file names.
 func StripSequence(fileName string) string {
 	if fileName == "" {
@@ -66,13 +61,3 @@ func AbsPrefix(fileName string, stripSequence bool) string {
 
 	return filepath.Join(filepath.Dir(fileName), BasePrefix(fileName, stripSequence))
 }
-
-// RelatedFilePathPrefix returns the absolute file path and name prefix without file extensions and media file
-// suffixes to be ignored for comparison, see https://github.com/photoprism/photoprism/issues/2983.
-func RelatedFilePathPrefix(fileName string, stripSequence bool) string {
-	if fileName == "" {
-		return ""
-	}
-
-	return RelatedMediaFileSuffix.ReplaceAllString(AbsPrefix(fileName, stripSequence), "")
-}

pkg/fs/filepath_test.go

@@ -130,6 +130,10 @@ func TestAbsPrefix(t *testing.T) {
 		assert.Equal(t, "", AbsPrefix("", true))
 		assert.Equal(t, "", AbsPrefix("", false))
 	})
+	t.Run("IMG_4120", func(t *testing.T) {
+		assert.Equal(t, "/foo/bar/IMG_4120", AbsPrefix("/foo/bar/IMG_4120.JPG", false))
+		assert.Equal(t, "/foo/bar/IMG_E4120", AbsPrefix("/foo/bar/IMG_E4120.JPG", false))
+	})
 	t.Run("Test copy 3.jpg", func(t *testing.T) {
 		result := AbsPrefix("/testdata/Test (4).jpg", true)
 
@@ -140,50 +144,11 @@ func TestAbsPrefix(t *testing.T) {
 
 		assert.Equal(t, "/testdata/Test (4)", result)
 	})
-}
-
-func TestRelatedFilePathPrefix(t *testing.T) {
-	t.Run("Empty", func(t *testing.T) {
-		assert.Equal(t, "", RelatedFilePathPrefix("", true))
-		assert.Equal(t, "", RelatedFilePathPrefix("", false))
-	})
-	t.Run("IMG_4120", func(t *testing.T) {
-		assert.Equal(t, "/foo/bar/IMG_4120", RelatedFilePathPrefix("/foo/bar/IMG_4120.JPG", false))
-		assert.Equal(t, "/foo/bar/IMG_E4120", RelatedFilePathPrefix("/foo/bar/IMG_E4120.JPG", false))
-	})
-	t.Run("LivePhoto", func(t *testing.T) {
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC.MOV", false))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC.MOV", true))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_HevC", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_HEVC.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_HEVC.MOV", true))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_hevc.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722_hevc_", RelatedFilePathPrefix("/foo/bar/IMG_1722_hevc_.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_HEVC.AVC", true))
-		assert.Equal(t, "/foo/bar/IMG_1722_MOV", RelatedFilePathPrefix("/foo/bar/IMG_1722_MOV.MOV", true))
-		assert.Equal(t, "/foo/bar/IMG_1722_AVC", RelatedFilePathPrefix("/foo/bar/IMG_1722_AVC.MOV", true))
-
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC.JPEG", false))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC.JPEG", true))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC (1).JPEG", true))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_HEVC (2).JPEG", true))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_JPEG (1).JPEG", true))
-		assert.Equal(t, "IMG_1722", RelatedFilePathPrefix("IMG_1722_JPG (2).JPEG", true))
-		assert.Equal(t, "IMG_1722_JPG (2)", RelatedFilePathPrefix("IMG_1722_JPG (2).JPEG", false))
-		assert.Equal(t, "IMG_1722_AVC", RelatedFilePathPrefix("IMG_1722_AVC (3).JPEG", true))
-		assert.Equal(t, "IMG_1722_AVC (3)", RelatedFilePathPrefix("IMG_1722_AVC (3).JPEG", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_Jpeg", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_JPEG.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_JPEG.MOV", true))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_jpeg.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722_jpeg_", RelatedFilePathPrefix("/foo/bar/IMG_1722_jpeg_.MOV", false))
-		assert.Equal(t, "/foo/bar/IMG_1722", RelatedFilePathPrefix("/foo/bar/IMG_1722_JPEG.JPEG", false))
-	})
 	t.Run("Sequence", func(t *testing.T) {
-		assert.Equal(t, "/foo/bar/Test", RelatedFilePathPrefix("/foo/bar/Test (4).jpg", true))
-		assert.Equal(t, "/foo/bar/Test (4)", RelatedFilePathPrefix("/foo/bar/Test (4).jpg", false))
+		assert.Equal(t, "/foo/bar/Test", AbsPrefix("/foo/bar/Test (4).jpg", true))
+		assert.Equal(t, "/foo/bar/Test (4)", AbsPrefix("/foo/bar/Test (4).jpg", false))
 	})
 	t.Run("LowerCase", func(t *testing.T) {
-		assert.Equal(t, "/foo/bar/IMG_E4120", RelatedFilePathPrefix("/foo/bar/IMG_E4120.JPG", false))
+		assert.Equal(t, "/foo/bar/IMG_E4120", AbsPrefix("/foo/bar/IMG_E4120.JPG", false))
 	})
 }

pkg/list/join.go

@@ -0,0 +1,18 @@
+package list
+
+// Join combines two lists without adding duplicates.
+func Join(list []string, join []string) []string {
+	if len(join) == 0 {
+		return list
+	} else if len(list) == 0 {
+		return join
+	}
+
+	for j := range join {
+		if Excludes(list, join[j]) {
+			list = append(list, join[j])
+		}
+	}
+
+	return list
+}

pkg/list/join_test.go

@@ -0,0 +1,23 @@
+package list
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoin(t *testing.T) {
+	assert.Equal(t, []string{""}, Join([]string{}, []string{""}))
+	assert.Equal(t, []string{"bar"}, Join([]string{}, []string{"bar"}))
+	assert.Equal(t, []string{""}, Join([]string{""}, []string{}))
+	assert.Equal(t, []string{"bar"}, Join([]string{"bar"}, []string{}))
+	assert.Equal(t, []string{"foo", "bar"}, Join([]string{"foo", "bar"}, []string{""}))
+	assert.Equal(t, []string{"foo", "bar"}, Join([]string{"foo", "bar"}, []string{"foo"}))
+	assert.Equal(t, []string{"foo", "bar", "zzz"}, Join([]string{"foo", "bar"}, []string{"zzz"}))
+	assert.Equal(t, []string{"foo", "bar", " "}, Join([]string{"foo", "bar"}, []string{" "}))
+	assert.Equal(t, []string{"foo", "bar", "645656"}, Join([]string{"foo", "bar"}, []string{"645656"}))
+	assert.Equal(t, []string{"foo", "bar ", "foo ", "baz", "bar"}, Join([]string{"foo", "bar ", "foo ", "baz"}, []string{"bar"}))
+	assert.Equal(t, []string{"foo", "bar", "foo ", "baz", "bar "}, Join([]string{"foo", "bar", "foo ", "baz"}, []string{"bar "}))
+	assert.Equal(t, []string{"bar", "baz", "foo", "bar ", "foo "}, Join([]string{"bar", "baz"}, []string{"foo", "bar ", "foo ", "baz"}))
+	assert.Equal(t, []string{"bar", "foo", "foo ", "baz"}, Join([]string{"bar"}, []string{"foo", "bar", "foo ", "baz"}))
+}