Backend: Use fastwalk.Walk instead of filepath.Walk in tests #267
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
parent
f5a1d76748
commit
44abbfc43a
3
go.mod
3
go.mod
|
@ -30,6 +30,7 @@ require (
|
||||||
github.com/guregu/null v3.4.0+incompatible // indirect
|
github.com/guregu/null v3.4.0+incompatible // indirect
|
||||||
github.com/jinzhu/gorm v1.9.5
|
github.com/jinzhu/gorm v1.9.5
|
||||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
|
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
|
||||||
|
github.com/karrick/godirwalk v1.15.6 // indirect
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||||
github.com/kr/pretty v0.1.0 // indirect
|
github.com/kr/pretty v0.1.0 // indirect
|
||||||
github.com/leandro-lugaresi/hub v1.1.0
|
github.com/leandro-lugaresi/hub v1.1.0
|
||||||
|
@ -78,7 +79,7 @@ require (
|
||||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
|
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
|
||||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e // indirect
|
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e // indirect
|
||||||
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
|
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
|
||||||
golang.org/x/tools v0.0.0-20200401192744-099440627f01 // indirect
|
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d
|
||||||
gopkg.in/stretchr/testify.v1 v1.2.2 // indirect
|
gopkg.in/stretchr/testify.v1 v1.2.2 // indirect
|
||||||
gopkg.in/ugjka/go-tz.v2 v2.0.9
|
gopkg.in/ugjka/go-tz.v2 v2.0.9
|
||||||
gopkg.in/yaml.v2 v2.2.8
|
gopkg.in/yaml.v2 v2.2.8
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -202,6 +202,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
|
||||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
|
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
|
||||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||||
|
github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
|
||||||
|
github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||||
|
@ -495,6 +497,8 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.0.0-20200401192744-099440627f01 h1:ysQJ/fU6laLOZJseIeOqXl6Mo+lw5z6b7QHnmUKjW+k=
|
golang.org/x/tools v0.0.0-20200401192744-099440627f01 h1:ysQJ/fU6laLOZJseIeOqXl6Mo+lw5z6b7QHnmUKjW+k=
|
||||||
golang.org/x/tools v0.0.0-20200401192744-099440627f01/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
golang.org/x/tools v0.0.0-20200401192744-099440627f01/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY=
|
||||||
|
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/photoprism/photoprism/pkg/fastwalk"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -60,12 +61,8 @@ func TestIsSafe(t *testing.T) {
|
||||||
"zebra_green_brown.jpg": {0.24, 0.01, 0.73, 0.004, 0.001},
|
"zebra_green_brown.jpg": {0.24, 0.01, 0.73, 0.004, 0.001},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := filepath.Walk("testdata", func(filename string, fileInfo os.FileInfo, err error) error {
|
if err := fastwalk.Walk("testdata", func(filename string, info os.FileMode) error {
|
||||||
if err != nil {
|
if info.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,10 +91,8 @@ func TestIsSafe(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
if err != nil {
|
|
||||||
t.Log(err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,14 +8,14 @@ import (
|
||||||
|
|
||||||
"github.com/photoprism/photoprism/internal/config"
|
"github.com/photoprism/photoprism/internal/config"
|
||||||
"github.com/photoprism/photoprism/pkg/colors"
|
"github.com/photoprism/photoprism/pkg/colors"
|
||||||
|
"github.com/photoprism/photoprism/pkg/fastwalk"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMediaFile_Colors_Testdata(t *testing.T) {
|
func TestMediaFile_Colors_Testdata(t *testing.T) {
|
||||||
conf := config.TestConfig()
|
conf := config.TestConfig()
|
||||||
|
|
||||||
thumbsPath := conf.CachePath() + "/_tmp"
|
thumbsPath := os.TempDir() + "/TestMediaFile_Colors_Testdata"
|
||||||
|
|
||||||
defer os.RemoveAll(thumbsPath)
|
defer os.RemoveAll(thumbsPath)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -54,12 +54,8 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := filepath.Walk(conf.ExamplesPath(), func(filename string, fileInfo os.FileInfo, err error) error {
|
if err := fastwalk.Walk(conf.ExamplesPath(), func(filename string, info os.FileMode) error {
|
||||||
if err != nil {
|
if info.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,10 +83,8 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
if err != nil {
|
|
||||||
t.Log(err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
196
pkg/fastwalk/fastwalk.go
Normal file
196
pkg/fastwalk/fastwalk.go
Normal file
|
@ -0,0 +1,196 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Package fastwalk provides a faster version of filepath.Walk for file system
|
||||||
|
// scanning tools.
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
|
||||||
|
// symlink named in the call may be traversed.
|
||||||
|
var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
|
||||||
|
|
||||||
|
// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
|
||||||
|
// callback should not be called for any other files in the current directory.
|
||||||
|
// Child directories will still be traversed.
|
||||||
|
var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
|
||||||
|
|
||||||
|
// Walk is a faster implementation of filepath.Walk.
|
||||||
|
//
|
||||||
|
// filepath.Walk's design necessarily calls os.Lstat on each file,
|
||||||
|
// even if the caller needs less info.
|
||||||
|
// Many tools need only the type of each file.
|
||||||
|
// On some platforms, this information is provided directly by the readdir
|
||||||
|
// system call, avoiding the need to stat each file individually.
|
||||||
|
// fastwalk_unix.go contains a fork of the syscall routines.
|
||||||
|
//
|
||||||
|
// See golang.org/issue/16399
|
||||||
|
//
|
||||||
|
// Walk walks the file tree rooted at root, calling walkFn for
|
||||||
|
// each file or directory in the tree, including root.
|
||||||
|
//
|
||||||
|
// If fastWalk returns filepath.SkipDir, the directory is skipped.
|
||||||
|
//
|
||||||
|
// Unlike filepath.Walk:
|
||||||
|
// * file stat calls must be done by the user.
|
||||||
|
// The only provided metadata is the file type, which does not include
|
||||||
|
// any permission bits.
|
||||||
|
// * multiple goroutines stat the filesystem concurrently. The provided
|
||||||
|
// walkFn must be safe for concurrent use.
|
||||||
|
// * fastWalk can follow symlinks if walkFn returns the TraverseLink
|
||||||
|
// sentinel error. It is the walkFn's responsibility to prevent
|
||||||
|
// fastWalk from going into symlink cycles.
|
||||||
|
func Walk(root string, walkFn func(path string, typ os.FileMode) error) error {
|
||||||
|
// TODO(bradfitz): make numWorkers configurable? We used a
|
||||||
|
// minimum of 4 to give the kernel more info about multiple
|
||||||
|
// things we want, in hopes its I/O scheduling can take
|
||||||
|
// advantage of that. Hopefully most are in cache. Maybe 4 is
|
||||||
|
// even too low of a minimum. Profile more.
|
||||||
|
numWorkers := 4
|
||||||
|
if n := runtime.NumCPU(); n > numWorkers {
|
||||||
|
numWorkers = n
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make sure to wait for all workers to finish, otherwise
|
||||||
|
// walkFn could still be called after returning. This Wait call
|
||||||
|
// runs after close(e.donec) below.
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
defer wg.Wait()
|
||||||
|
|
||||||
|
w := &walker{
|
||||||
|
fn: walkFn,
|
||||||
|
enqueuec: make(chan walkItem, numWorkers), // buffered for performance
|
||||||
|
workc: make(chan walkItem, numWorkers), // buffered for performance
|
||||||
|
donec: make(chan struct{}),
|
||||||
|
|
||||||
|
// buffered for correctness & not leaking goroutines:
|
||||||
|
resc: make(chan error, numWorkers),
|
||||||
|
}
|
||||||
|
defer close(w.donec)
|
||||||
|
|
||||||
|
for i := 0; i < numWorkers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go w.doWork(&wg)
|
||||||
|
}
|
||||||
|
todo := []walkItem{{dir: root}}
|
||||||
|
out := 0
|
||||||
|
for {
|
||||||
|
workc := w.workc
|
||||||
|
var workItem walkItem
|
||||||
|
if len(todo) == 0 {
|
||||||
|
workc = nil
|
||||||
|
} else {
|
||||||
|
workItem = todo[len(todo)-1]
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case workc <- workItem:
|
||||||
|
todo = todo[:len(todo)-1]
|
||||||
|
out++
|
||||||
|
case it := <-w.enqueuec:
|
||||||
|
todo = append(todo, it)
|
||||||
|
case err := <-w.resc:
|
||||||
|
out--
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if out == 0 && len(todo) == 0 {
|
||||||
|
// It's safe to quit here, as long as the buffered
|
||||||
|
// enqueue channel isn't also readable, which might
|
||||||
|
// happen if the worker sends both another unit of
|
||||||
|
// work and its result before the other select was
|
||||||
|
// scheduled and both w.resc and w.enqueuec were
|
||||||
|
// readable.
|
||||||
|
select {
|
||||||
|
case it := <-w.enqueuec:
|
||||||
|
todo = append(todo, it)
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doWork reads directories as instructed (via workc) and runs the
|
||||||
|
// user's callback function.
|
||||||
|
func (w *walker) doWork(wg *sync.WaitGroup) {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-w.donec:
|
||||||
|
return
|
||||||
|
case it := <-w.workc:
|
||||||
|
select {
|
||||||
|
case <-w.donec:
|
||||||
|
return
|
||||||
|
case w.resc <- w.walk(it.dir, !it.callbackDone):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type walker struct {
|
||||||
|
fn func(path string, typ os.FileMode) error
|
||||||
|
|
||||||
|
donec chan struct{} // closed on fastWalk's return
|
||||||
|
workc chan walkItem // to workers
|
||||||
|
enqueuec chan walkItem // from workers
|
||||||
|
resc chan error // from workers
|
||||||
|
}
|
||||||
|
|
||||||
|
type walkItem struct {
|
||||||
|
dir string
|
||||||
|
callbackDone bool // callback already called; don't do it again
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *walker) enqueue(it walkItem) {
|
||||||
|
select {
|
||||||
|
case w.enqueuec <- it:
|
||||||
|
case <-w.donec:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
|
||||||
|
joined := dirName + string(os.PathSeparator) + baseName
|
||||||
|
if typ == os.ModeDir {
|
||||||
|
w.enqueue(walkItem{dir: joined})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err := w.fn(joined, typ)
|
||||||
|
if typ == os.ModeSymlink {
|
||||||
|
if err == ErrTraverseLink {
|
||||||
|
// Set callbackDone so we don't call it twice for both the
|
||||||
|
// symlink-as-symlink and the symlink-as-directory later:
|
||||||
|
w.enqueue(walkItem{dir: joined, callbackDone: true})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err == filepath.SkipDir {
|
||||||
|
// Permit SkipDir on symlinks too.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *walker) walk(root string, runUserCallback bool) error {
|
||||||
|
if runUserCallback {
|
||||||
|
err := w.fn(root, os.ModeDir)
|
||||||
|
if err == filepath.SkipDir {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return readDir(root, w.onDirEnt)
|
||||||
|
}
|
13
pkg/fastwalk/fastwalk_dirent_fileno.go
Normal file
13
pkg/fastwalk/fastwalk_dirent_fileno.go
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build freebsd openbsd netbsd
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func direntInode(dirent *syscall.Dirent) uint64 {
|
||||||
|
return uint64(dirent.Fileno)
|
||||||
|
}
|
14
pkg/fastwalk/fastwalk_dirent_ino.go
Normal file
14
pkg/fastwalk/fastwalk_dirent_ino.go
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build linux darwin
|
||||||
|
// +build !appengine
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func direntInode(dirent *syscall.Dirent) uint64 {
|
||||||
|
return uint64(dirent.Ino)
|
||||||
|
}
|
13
pkg/fastwalk/fastwalk_dirent_namlen_bsd.go
Normal file
13
pkg/fastwalk/fastwalk_dirent_namlen_bsd.go
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
// Copyright 2018 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build darwin freebsd openbsd netbsd
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func direntNamlen(dirent *syscall.Dirent) uint64 {
|
||||||
|
return uint64(dirent.Namlen)
|
||||||
|
}
|
29
pkg/fastwalk/fastwalk_dirent_namlen_linux.go
Normal file
29
pkg/fastwalk/fastwalk_dirent_namlen_linux.go
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
// Copyright 2018 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build linux
|
||||||
|
// +build !appengine
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
func direntNamlen(dirent *syscall.Dirent) uint64 {
|
||||||
|
const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name))
|
||||||
|
nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
|
||||||
|
const nameBufLen = uint16(len(nameBuf))
|
||||||
|
limit := dirent.Reclen - fixedHdr
|
||||||
|
if limit > nameBufLen {
|
||||||
|
limit = nameBufLen
|
||||||
|
}
|
||||||
|
nameLen := bytes.IndexByte(nameBuf[:limit], 0)
|
||||||
|
if nameLen < 0 {
|
||||||
|
panic("failed to find terminating 0 byte in dirent")
|
||||||
|
}
|
||||||
|
return uint64(nameLen)
|
||||||
|
}
|
37
pkg/fastwalk/fastwalk_portable.go
Normal file
37
pkg/fastwalk/fastwalk_portable.go
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// readDir calls fn for each directory entry in dirName.
|
||||||
|
// It does not descend into directories or follow symlinks.
|
||||||
|
// If fn returns a non-nil error, readDir returns with that error
|
||||||
|
// immediately.
|
||||||
|
func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
|
||||||
|
fis, err := ioutil.ReadDir(dirName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
skipFiles := false
|
||||||
|
for _, fi := range fis {
|
||||||
|
if fi.Mode().IsRegular() && skipFiles {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := fn(dirName, fi.Name(), fi.Mode()&os.ModeType); err != nil {
|
||||||
|
if err == ErrSkipFiles {
|
||||||
|
skipFiles = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
239
pkg/fastwalk/fastwalk_test.go
Normal file
239
pkg/fastwalk/fastwalk_test.go
Normal file
|
@ -0,0 +1,239 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package fastwalk_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/photoprism/photoprism/pkg/fastwalk"
|
||||||
|
)
|
||||||
|
|
||||||
|
func formatFileModes(m map[string]os.FileMode) string {
|
||||||
|
var keys []string
|
||||||
|
for k := range m {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
for _, k := range keys {
|
||||||
|
fmt.Fprintf(&buf, "%-20s: %v\n", k, m[k])
|
||||||
|
}
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func testFastWalk(t *testing.T, files map[string]string, callback func(path string, typ os.FileMode) error, want map[string]os.FileMode) {
|
||||||
|
tempdir, err := ioutil.TempDir("", "test-fast-walk")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempdir)
|
||||||
|
for path, contents := range files {
|
||||||
|
file := filepath.Join(tempdir, "/src", path)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if strings.HasPrefix(contents, "LINK:") {
|
||||||
|
err = os.Symlink(strings.TrimPrefix(contents, "LINK:"), file)
|
||||||
|
} else {
|
||||||
|
err = ioutil.WriteFile(file, []byte(contents), 0644)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got := map[string]os.FileMode{}
|
||||||
|
var mu sync.Mutex
|
||||||
|
if err := fastwalk.Walk(tempdir, func(path string, typ os.FileMode) error {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if !strings.HasPrefix(path, tempdir) {
|
||||||
|
t.Fatalf("bogus prefix on %q, expect %q", path, tempdir)
|
||||||
|
}
|
||||||
|
key := filepath.ToSlash(strings.TrimPrefix(path, tempdir))
|
||||||
|
if old, dup := got[key]; dup {
|
||||||
|
t.Fatalf("callback called twice for key %q: %v -> %v", key, old, typ)
|
||||||
|
}
|
||||||
|
got[key] = typ
|
||||||
|
return callback(path, typ)
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("callback returned: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("walk mismatch.\n got:\n%v\nwant:\n%v", formatFileModes(got), formatFileModes(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_Basic(t *testing.T) {
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
"foo/foo.go": "one",
|
||||||
|
"bar/bar.go": "two",
|
||||||
|
"skip/skip.go": "skip",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/bar": os.ModeDir,
|
||||||
|
"/src/bar/bar.go": 0,
|
||||||
|
"/src/foo": os.ModeDir,
|
||||||
|
"/src/foo/foo.go": 0,
|
||||||
|
"/src/skip": os.ModeDir,
|
||||||
|
"/src/skip/skip.go": 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_LongFileName(t *testing.T) {
|
||||||
|
longFileName := strings.Repeat("x", 255)
|
||||||
|
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
longFileName: "one",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/" + longFileName: 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_Symlink(t *testing.T) {
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "windows", "plan9":
|
||||||
|
t.Skipf("skipping on %s", runtime.GOOS)
|
||||||
|
}
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
"foo/foo.go": "one",
|
||||||
|
"bar/bar.go": "LINK:../foo.go",
|
||||||
|
"symdir": "LINK:foo",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/bar": os.ModeDir,
|
||||||
|
"/src/bar/bar.go": os.ModeSymlink,
|
||||||
|
"/src/foo": os.ModeDir,
|
||||||
|
"/src/foo/foo.go": 0,
|
||||||
|
"/src/symdir": os.ModeSymlink,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_SkipDir(t *testing.T) {
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
"foo/foo.go": "one",
|
||||||
|
"bar/bar.go": "two",
|
||||||
|
"skip/skip.go": "skip",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
if typ == os.ModeDir && strings.HasSuffix(path, "skip") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/bar": os.ModeDir,
|
||||||
|
"/src/bar/bar.go": 0,
|
||||||
|
"/src/foo": os.ModeDir,
|
||||||
|
"/src/foo/foo.go": 0,
|
||||||
|
"/src/skip": os.ModeDir,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_SkipFiles(t *testing.T) {
|
||||||
|
// Directory iteration order is undefined, so there's no way to know
|
||||||
|
// which file to expect until the walk happens. Rather than mess
|
||||||
|
// with the test infrastructure, just mutate want.
|
||||||
|
var mu sync.Mutex
|
||||||
|
want := map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/zzz": os.ModeDir,
|
||||||
|
"/src/zzz/c.go": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
"a_skipfiles.go": "a",
|
||||||
|
"b_skipfiles.go": "b",
|
||||||
|
"zzz/c.go": "c",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
if strings.HasSuffix(path, "_skipfiles.go") {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
want["/src/"+filepath.Base(path)] = 0
|
||||||
|
return fastwalk.ErrSkipFiles
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
want)
|
||||||
|
if len(want) != 5 {
|
||||||
|
t.Errorf("saw too many files: wanted 5, got %v (%v)", len(want), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFastWalk_TraverseSymlink(t *testing.T) {
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "windows", "plan9":
|
||||||
|
t.Skipf("skipping on %s", runtime.GOOS)
|
||||||
|
}
|
||||||
|
|
||||||
|
testFastWalk(t, map[string]string{
|
||||||
|
"foo/foo.go": "one",
|
||||||
|
"bar/bar.go": "two",
|
||||||
|
"skip/skip.go": "skip",
|
||||||
|
"symdir": "LINK:foo",
|
||||||
|
},
|
||||||
|
func(path string, typ os.FileMode) error {
|
||||||
|
if typ == os.ModeSymlink {
|
||||||
|
return fastwalk.ErrTraverseLink
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
map[string]os.FileMode{
|
||||||
|
"": os.ModeDir,
|
||||||
|
"/src": os.ModeDir,
|
||||||
|
"/src/bar": os.ModeDir,
|
||||||
|
"/src/bar/bar.go": 0,
|
||||||
|
"/src/foo": os.ModeDir,
|
||||||
|
"/src/foo/foo.go": 0,
|
||||||
|
"/src/skip": os.ModeDir,
|
||||||
|
"/src/skip/skip.go": 0,
|
||||||
|
"/src/symdir": os.ModeSymlink,
|
||||||
|
"/src/symdir/foo.go": 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
var benchDir = flag.String("benchdir", runtime.GOROOT(), "The directory to scan for BenchmarkFastWalk")
|
||||||
|
|
||||||
|
func BenchmarkFastWalk(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := fastwalk.Walk(*benchDir, func(path string, typ os.FileMode) error { return nil })
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
128
pkg/fastwalk/fastwalk_unix.go
Normal file
128
pkg/fastwalk/fastwalk_unix.go
Normal file
|
@ -0,0 +1,128 @@
|
||||||
|
// Copyright 2016 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build linux darwin freebsd openbsd netbsd
|
||||||
|
// +build !appengine
|
||||||
|
|
||||||
|
package fastwalk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const blockSize = 8 << 10
|
||||||
|
|
||||||
|
// unknownFileMode is a sentinel (and bogus) os.FileMode
|
||||||
|
// value used to represent a syscall.DT_UNKNOWN Dirent.Type.
|
||||||
|
const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice
|
||||||
|
|
||||||
|
func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
|
||||||
|
fd, err := syscall.Open(dirName, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
return &os.PathError{Op: "open", Path: dirName, Err: err}
|
||||||
|
}
|
||||||
|
defer syscall.Close(fd)
|
||||||
|
|
||||||
|
// The buffer must be at least a block long.
|
||||||
|
buf := make([]byte, blockSize) // stack-allocated; doesn't escape
|
||||||
|
bufp := 0 // starting read position in buf
|
||||||
|
nbuf := 0 // end valid data in buf
|
||||||
|
skipFiles := false
|
||||||
|
for {
|
||||||
|
if bufp >= nbuf {
|
||||||
|
bufp = 0
|
||||||
|
nbuf, err = syscall.ReadDirent(fd, buf)
|
||||||
|
if err != nil {
|
||||||
|
return os.NewSyscallError("readdirent", err)
|
||||||
|
}
|
||||||
|
if nbuf <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
consumed, name, typ := parseDirEnt(buf[bufp:nbuf])
|
||||||
|
bufp += consumed
|
||||||
|
if name == "" || name == "." || name == ".." {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Fallback for filesystems (like old XFS) that don't
|
||||||
|
// support Dirent.Type and have DT_UNKNOWN (0) there
|
||||||
|
// instead.
|
||||||
|
if typ == unknownFileMode {
|
||||||
|
fi, err := os.Lstat(dirName + "/" + name)
|
||||||
|
if err != nil {
|
||||||
|
// It got deleted in the meantime.
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
typ = fi.Mode() & os.ModeType
|
||||||
|
}
|
||||||
|
if skipFiles && typ.IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := fn(dirName, name, typ); err != nil {
|
||||||
|
if err == ErrSkipFiles {
|
||||||
|
skipFiles = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) {
|
||||||
|
// golang.org/issue/37269
|
||||||
|
dirent := &syscall.Dirent{}
|
||||||
|
copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf)
|
||||||
|
if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
|
||||||
|
panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v))
|
||||||
|
}
|
||||||
|
if len(buf) < int(dirent.Reclen) {
|
||||||
|
panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen))
|
||||||
|
}
|
||||||
|
consumed = int(dirent.Reclen)
|
||||||
|
if direntInode(dirent) == 0 { // File absent in directory.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch dirent.Type {
|
||||||
|
case syscall.DT_REG:
|
||||||
|
typ = 0
|
||||||
|
case syscall.DT_DIR:
|
||||||
|
typ = os.ModeDir
|
||||||
|
case syscall.DT_LNK:
|
||||||
|
typ = os.ModeSymlink
|
||||||
|
case syscall.DT_BLK:
|
||||||
|
typ = os.ModeDevice
|
||||||
|
case syscall.DT_FIFO:
|
||||||
|
typ = os.ModeNamedPipe
|
||||||
|
case syscall.DT_SOCK:
|
||||||
|
typ = os.ModeSocket
|
||||||
|
case syscall.DT_UNKNOWN:
|
||||||
|
typ = unknownFileMode
|
||||||
|
default:
|
||||||
|
// Skip weird things.
|
||||||
|
// It's probably a DT_WHT (http://lwn.net/Articles/325369/)
|
||||||
|
// or something. Revisit if/when this package is moved outside
|
||||||
|
// of goimports. goimports only cares about regular files,
|
||||||
|
// symlinks, and directories.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
|
||||||
|
nameLen := direntNamlen(dirent)
|
||||||
|
|
||||||
|
// Special cases for common things:
|
||||||
|
if nameLen == 1 && nameBuf[0] == '.' {
|
||||||
|
name = "."
|
||||||
|
} else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' {
|
||||||
|
name = ".."
|
||||||
|
} else {
|
||||||
|
name = string(nameBuf[:nameLen])
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
Loading…
Reference in a new issue