From 6847b8b5f9ac7a01e7c07e43f4360e9a504727a3 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Thu, 16 Jul 2020 15:43:23 +0200 Subject: [PATCH] Complete file browser with symlink support and full folder / file names (#412) * Backend: Code clean-up Signed-off-by: Michael Mayer * File Browser: Show complete, original file and folder names #408 Signed-off-by: Michael Mayer * File Browser: Follow symlinks #201 #403 #407 Warning: Following symlinks can make folder lists non-deterministic Signed-off-by: Michael Mayer --- frontend/src/pages/library/files.vue | 9 +-- internal/api/folder_test.go | 8 +-- internal/entity/file.go | 2 +- internal/form/photo.go | 2 +- internal/maps/location_test.go | 4 +- internal/query/folders.go | 2 +- pkg/fs/dirs.go | 42 +++++++++++--- pkg/fs/dirs_test.go | 55 +++++++++++++++++-- pkg/fs/ignore_test.go | 1 + .../directory/subdirectory/bar/helloworld.md | 1 + pkg/fs/testdata/linked/photoprism | 1 + pkg/txt/convert.go | 2 - 12 files changed, 98 insertions(+), 31 deletions(-) create mode 100644 pkg/fs/testdata/directory/subdirectory/bar/helloworld.md create mode 120000 pkg/fs/testdata/linked/photoprism diff --git a/frontend/src/pages/library/files.vue b/frontend/src/pages/library/files.vue index 5901b40e6..25648c106 100644 --- a/frontend/src/pages/library/files.vue +++ b/frontend/src/pages/library/files.vue @@ -91,7 +91,7 @@

@@ -103,13 +103,10 @@

-
- /{{ model.Path }} -
-
+
Folder
diff --git a/internal/api/folder_test.go b/internal/api/folder_test.go index 2a45e0b19..94bc1f665 100644 --- a/internal/api/folder_test.go +++ b/internal/api/folder_test.go @@ -13,7 +13,7 @@ func TestGetFoldersOriginals(t *testing.T) { t.Run("flat", func(t *testing.T) { app, router, conf := NewApiTest() _ = conf.CreateDirectories() - expected, err := fs.Dirs(conf.OriginalsPath(), false) + expected, err := fs.Dirs(conf.OriginalsPath(), false, true) if err != nil { t.Fatal(err) @@ -56,7 +56,7 @@ func TestGetFoldersOriginals(t *testing.T) { t.Run("recursive", func(t *testing.T) { app, router, conf := NewApiTest() _ = conf.CreateDirectories() - expected, err := fs.Dirs(conf.OriginalsPath(), true) + expected, err := fs.Dirs(conf.OriginalsPath(), true, true) if err != nil { t.Fatal(err) @@ -96,7 +96,7 @@ func TestGetFoldersImport(t *testing.T) { t.Run("flat", func(t *testing.T) { app, router, conf := NewApiTest() _ = conf.CreateDirectories() - expected, err := fs.Dirs(conf.ImportPath(), false) + expected, err := fs.Dirs(conf.ImportPath(), false, true) if err != nil { t.Fatal(err) @@ -140,7 +140,7 @@ func TestGetFoldersImport(t *testing.T) { t.Run("recursive", func(t *testing.T) { app, router, conf := NewApiTest() _ = conf.CreateDirectories() - expected, err := fs.Dirs(conf.ImportPath(), true) + expected, err := fs.Dirs(conf.ImportPath(), true, true) if err != nil { t.Fatal(err) diff --git a/internal/entity/file.go b/internal/entity/file.go index 0fbea45e5..70afdf07a 100644 --- a/internal/entity/file.go +++ b/internal/entity/file.go @@ -226,5 +226,5 @@ func (m *File) Panorama() bool { return false } - return m.FileProjection != ProjectionDefault || m.FileWidth / m.FileHeight > 2 + return m.FileProjection != ProjectionDefault || m.FileWidth/m.FileHeight > 2 } diff --git a/internal/form/photo.go b/internal/form/photo.go index f4fe5c7ab..ef1155722 100644 --- a/internal/form/photo.go +++ b/internal/form/photo.go @@ -36,7 +36,7 @@ type Photo struct { PhotoPrivate bool `json:"Private"` PhotoReview bool `json:"Review"` PhotoScan bool `json:"Scan"` - PhotoPanorama bool `json:"Panorama"` + PhotoPanorama bool `json:"Panorama"` PhotoAltitude int `json:"Altitude"` PhotoLat float32 `json:"Lat"` PhotoLng float32 `json:"Lng"` diff --git a/internal/maps/location_test.go b/internal/maps/location_test.go index f3d934436..b3eab81cc 100644 --- a/internal/maps/location_test.go +++ b/internal/maps/location_test.go @@ -176,10 +176,10 @@ func TestLocation_Assign(t *testing.T) { lat := -21.976301666666668 lng := 49.148046666666666 id := s2.Token(lat, lng) - log.Printf("ID: %s", id) + // log.Printf("ID: %s", id) o, err := places.FindLocation(id) - log.Printf("Output: %+v", o) + // log.Printf("Output: %+v", o) if err != nil { t.Fatal(err) diff --git a/internal/query/folders.go b/internal/query/folders.go index c509fab45..fde245576 100644 --- a/internal/query/folders.go +++ b/internal/query/folders.go @@ -9,7 +9,7 @@ import ( // FoldersByPath returns a slice of folders in a given directory incl sub directories in recursive mode. func FoldersByPath(rootName, rootPath, path string, recursive bool) (folders entity.Folders, err error) { - dirs, err := fs.Dirs(filepath.Join(rootPath, path), recursive) + dirs, err := fs.Dirs(filepath.Join(rootPath, path), recursive, true) if err != nil { return folders, err diff --git a/pkg/fs/dirs.go b/pkg/fs/dirs.go index 1da07235a..e0d9ce5bd 100644 --- a/pkg/fs/dirs.go +++ b/pkg/fs/dirs.go @@ -88,25 +88,51 @@ var ImportPaths = []string{ "~/Import", } -func Dirs(root string, recursive bool) (result []string, err error) { +// Dirs returns a slice of directories in a path, optional recursively and with symlinks. +// +// Warning: Following symlinks can make the result non-deterministic and hard to test! +func Dirs(root string, recursive bool, followLinks bool) (result []string, err error) { result = []string{} ignore := NewIgnoreList(".ppignore", true, false) mutex := sync.Mutex{} - err = fastwalk.Walk(root, func(fileName string, info os.FileMode) error { - if info.IsDir() { + symlinks := make(map[string]bool) + symlinksMutex := sync.Mutex{} + + appendResult := func(fileName string) { + fileName = strings.Replace(fileName, root, "", 1) + mutex.Lock() + defer mutex.Unlock() + result = append(result, fileName) + } + + err = fastwalk.Walk(root, func(fileName string, typ os.FileMode) error { + if typ.IsDir() || typ == os.ModeSymlink && followLinks { if ignore.Ignore(fileName) { return filepath.SkipDir } if fileName != root { - mutex.Lock() - fileName = strings.Replace(fileName, root, "", 1) - result = append(result, fileName) - mutex.Unlock() - if !recursive { + appendResult(fileName) + return filepath.SkipDir + } else if typ != os.ModeSymlink { + appendResult(fileName) + + return nil + } else if resolved, err := filepath.EvalSymlinks(fileName); err == nil { + symlinksMutex.Lock() + defer symlinksMutex.Unlock() + + if _, ok := symlinks[resolved]; ok { + return filepath.SkipDir + } else { + symlinks[resolved] = true + appendResult(fileName) + } + + return fastwalk.ErrTraverseLink } } } diff --git a/pkg/fs/dirs_test.go b/pkg/fs/dirs_test.go index 68322dc58..d6f685682 100644 --- a/pkg/fs/dirs_test.go +++ b/pkg/fs/dirs_test.go @@ -9,27 +9,70 @@ import ( func TestDirs(t *testing.T) { t.Run("recursive", func(t *testing.T) { - result, err := Dirs("testdata", true) + result, err := Dirs("testdata", true, true) if err != nil { t.Fatal(err) } - expected := []string{"/directory", "/directory/subdirectory", "/linked"} + assert.Contains(t, result, "/directory") + assert.Contains(t, result, "/directory/subdirectory") + assert.Contains(t, result, "/linked") + assert.Contains(t, result, "/linked/photoprism") + assert.Contains(t, result, "/linked/photoprism/sub") + }) - assert.Equal(t, expected, result) + t.Run("recursive no-symlinks", func(t *testing.T) { + result, err := Dirs("testdata", true, false) + + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, result, "/directory") + assert.Contains(t, result, "/directory/subdirectory") + assert.Contains(t, result, "/linked") }) t.Run("non-recursive", func(t *testing.T) { - result, err := Dirs("testdata", false) + result, err := Dirs("testdata", false, true) if err != nil { t.Fatal(err) } - expected := []string{"/directory", "/linked"} + assert.Contains(t, result, "/directory") + assert.Contains(t, result, "/linked") + }) - assert.Equal(t, expected, result) + t.Run("non-recursive no-symlinks", func(t *testing.T) { + result, err := Dirs("testdata/directory/subdirectory", false, false) + + if err != nil { + t.Fatal(err) + } + assert.Contains(t, result, "/bar") + }) + + t.Run("non-recursive symlinks", func(t *testing.T) { + result, err := Dirs("testdata/linked", false, true) + + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, result, "/photoprism") + assert.Contains(t, result, "/self") + }) + + t.Run("no-result", func(t *testing.T) { + result, err := Dirs("testdata/linked", false, false) + + if err != nil { + t.Fatal(err) + } + + assert.Empty(t, result) }) } diff --git a/pkg/fs/ignore_test.go b/pkg/fs/ignore_test.go index 1320c8808..b1605e3ae 100644 --- a/pkg/fs/ignore_test.go +++ b/pkg/fs/ignore_test.go @@ -94,6 +94,7 @@ func TestIgnoreList_Ignored(t *testing.T) { expectIgnored := []string{ "testdata/directory/bar.txt", "testdata/directory/baz.xml", + "testdata/directory/subdirectory/bar", "testdata/directory/subdirectory/example.txt", "testdata/directory/subdirectory/foo.txt", } diff --git a/pkg/fs/testdata/directory/subdirectory/bar/helloworld.md b/pkg/fs/testdata/directory/subdirectory/bar/helloworld.md new file mode 100644 index 000000000..cc0be1e56 --- /dev/null +++ b/pkg/fs/testdata/directory/subdirectory/bar/helloworld.md @@ -0,0 +1 @@ +# Hello World! diff --git a/pkg/fs/testdata/linked/photoprism b/pkg/fs/testdata/linked/photoprism new file mode 120000 index 000000000..b9fb6acee --- /dev/null +++ b/pkg/fs/testdata/linked/photoprism @@ -0,0 +1 @@ +../.photoprism \ No newline at end of file diff --git a/pkg/txt/convert.go b/pkg/txt/convert.go index 7a30ee63e..2bcd7adbc 100644 --- a/pkg/txt/convert.go +++ b/pkg/txt/convert.go @@ -1,7 +1,6 @@ package txt import ( - "fmt" "regexp" "strconv" "strings" @@ -116,7 +115,6 @@ func Time(s string) (result time.Time) { time.UTC) } else if found := DatePathRegexp.Find(b); len(found) > 0 { // Is it a date path like "2020/01/03"? n := DateIntRegexp.FindAll(found, -1) - fmt.Println(n) if len(n) < 2 || len(n) > 3 { return result