Backend: Use fastwalk.Walk instead of filepath.Walk in tests #267

Signed-off-by: Michael Mayer <michael@liquidbytes.net>

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/guregu/null v3.4.0+incompatible // indirect
 	github.com/jinzhu/gorm v1.9.5
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/karrick/godirwalk v1.15.6 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/leandro-lugaresi/hub v1.1.0
@@ -78,7 +79,7 @@ require (
 	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
 	golang.org/x/sys v0.0.0-20200501052902-10377860bb8e // indirect
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
-	golang.org/x/tools v0.0.0-20200401192744-099440627f01 // indirect
+	golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d
 	gopkg.in/stretchr/testify.v1 v1.2.2 // indirect
 	gopkg.in/ugjka/go-tz.v2 v2.0.9
 	gopkg.in/yaml.v2 v2.2.8

go.sum

@@ -202,6 +202,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
+github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -495,6 +497,8 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200401192744-099440627f01 h1:ysQJ/fU6laLOZJseIeOqXl6Mo+lw5z6b7QHnmUKjW+k=
 golang.org/x/tools v0.0.0-20200401192744-099440627f01/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/nsfw/nsfw_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/photoprism/photoprism/pkg/fastwalk"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -60,12 +61,8 @@ func TestIsSafe(t *testing.T) {
 		"zebra_green_brown.jpg": {0.24, 0.01, 0.73, 0.004, 0.001},
 	}
 
-	err := filepath.Walk("testdata", func(filename string, fileInfo os.FileInfo, err error) error {
-		if err != nil {
-			return nil
-		}
-
-		if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
+	if err := fastwalk.Walk("testdata", func(filename string, info os.FileMode) error {
+		if info.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
 			return nil
 		}
 
@@ -94,10 +91,8 @@ func TestIsSafe(t *testing.T) {
 		})
 
 		return nil
-	})
-
-	if err != nil {
-		t.Log(err.Error())
+	}); err != nil {
+		t.Fatal(err)
 	}
 }
 

internal/photoprism/colors_test.go

@@ -8,14 +8,14 @@ import (
 
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/pkg/colors"
+	"github.com/photoprism/photoprism/pkg/fastwalk"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestMediaFile_Colors_Testdata(t *testing.T) {
 	conf := config.TestConfig()
 
-	thumbsPath := conf.CachePath() + "/_tmp"
-
+	thumbsPath := os.TempDir() + "/TestMediaFile_Colors_Testdata"
 	defer os.RemoveAll(thumbsPath)
 
 	/*
@@ -54,12 +54,8 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
 		},
 	}
 
-	err := filepath.Walk(conf.ExamplesPath(), func(filename string, fileInfo os.FileInfo, err error) error {
-		if err != nil {
-			return nil
-		}
-
-		if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
+	if err := fastwalk.Walk(conf.ExamplesPath(), func(filename string, info os.FileMode) error {
+		if info.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
 			return nil
 		}
 
@@ -87,10 +83,8 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
 		})
 
 		return nil
-	})
-
-	if err != nil {
-		t.Log(err.Error())
+	}); err != nil {
+		t.Fatal(err)
 	}
 }
 

pkg/fastwalk/fastwalk.go

@@ -0,0 +1,196 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package fastwalk provides a faster version of filepath.Walk for file system
+// scanning tools.
+package fastwalk
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+)
+
+// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
+// symlink named in the call may be traversed.
+var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
+
+// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
+// callback should not be called for any other files in the current directory.
+// Child directories will still be traversed.
+var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
+
+// Walk is a faster implementation of filepath.Walk.
+//
+// filepath.Walk's design necessarily calls os.Lstat on each file,
+// even if the caller needs less info.
+// Many tools need only the type of each file.
+// On some platforms, this information is provided directly by the readdir
+// system call, avoiding the need to stat each file individually.
+// fastwalk_unix.go contains a fork of the syscall routines.
+//
+// See golang.org/issue/16399
+//
+// Walk walks the file tree rooted at root, calling walkFn for
+// each file or directory in the tree, including root.
+//
+// If fastWalk returns filepath.SkipDir, the directory is skipped.
+//
+// Unlike filepath.Walk:
+//   * file stat calls must be done by the user.
+//     The only provided metadata is the file type, which does not include
+//     any permission bits.
+//   * multiple goroutines stat the filesystem concurrently. The provided
+//     walkFn must be safe for concurrent use.
+//   * fastWalk can follow symlinks if walkFn returns the TraverseLink
+//     sentinel error. It is the walkFn's responsibility to prevent
+//     fastWalk from going into symlink cycles.
+func Walk(root string, walkFn func(path string, typ os.FileMode) error) error {
+	// TODO(bradfitz): make numWorkers configurable? We used a
+	// minimum of 4 to give the kernel more info about multiple
+	// things we want, in hopes its I/O scheduling can take
+	// advantage of that. Hopefully most are in cache. Maybe 4 is
+	// even too low of a minimum. Profile more.
+	numWorkers := 4
+	if n := runtime.NumCPU(); n > numWorkers {
+		numWorkers = n
+	}
+
+	// Make sure to wait for all workers to finish, otherwise
+	// walkFn could still be called after returning. This Wait call
+	// runs after close(e.donec) below.
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	w := &walker{
+		fn:       walkFn,
+		enqueuec: make(chan walkItem, numWorkers), // buffered for performance
+		workc:    make(chan walkItem, numWorkers), // buffered for performance
+		donec:    make(chan struct{}),
+
+		// buffered for correctness & not leaking goroutines:
+		resc: make(chan error, numWorkers),
+	}
+	defer close(w.donec)
+
+	for i := 0; i < numWorkers; i++ {
+		wg.Add(1)
+		go w.doWork(&wg)
+	}
+	todo := []walkItem{{dir: root}}
+	out := 0
+	for {
+		workc := w.workc
+		var workItem walkItem
+		if len(todo) == 0 {
+			workc = nil
+		} else {
+			workItem = todo[len(todo)-1]
+		}
+		select {
+		case workc <- workItem:
+			todo = todo[:len(todo)-1]
+			out++
+		case it := <-w.enqueuec:
+			todo = append(todo, it)
+		case err := <-w.resc:
+			out--
+			if err != nil {
+				return err
+			}
+			if out == 0 && len(todo) == 0 {
+				// It's safe to quit here, as long as the buffered
+				// enqueue channel isn't also readable, which might
+				// happen if the worker sends both another unit of
+				// work and its result before the other select was
+				// scheduled and both w.resc and w.enqueuec were
+				// readable.
+				select {
+				case it := <-w.enqueuec:
+					todo = append(todo, it)
+				default:
+					return nil
+				}
+			}
+		}
+	}
+}
+
+// doWork reads directories as instructed (via workc) and runs the
+// user's callback function.
+func (w *walker) doWork(wg *sync.WaitGroup) {
+	defer wg.Done()
+	for {
+		select {
+		case <-w.donec:
+			return
+		case it := <-w.workc:
+			select {
+			case <-w.donec:
+				return
+			case w.resc <- w.walk(it.dir, !it.callbackDone):
+			}
+		}
+	}
+}
+
+type walker struct {
+	fn func(path string, typ os.FileMode) error
+
+	donec    chan struct{} // closed on fastWalk's return
+	workc    chan walkItem // to workers
+	enqueuec chan walkItem // from workers
+	resc     chan error    // from workers
+}
+
+type walkItem struct {
+	dir          string
+	callbackDone bool // callback already called; don't do it again
+}
+
+func (w *walker) enqueue(it walkItem) {
+	select {
+	case w.enqueuec <- it:
+	case <-w.donec:
+	}
+}
+
+func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
+	joined := dirName + string(os.PathSeparator) + baseName
+	if typ == os.ModeDir {
+		w.enqueue(walkItem{dir: joined})
+		return nil
+	}
+
+	err := w.fn(joined, typ)
+	if typ == os.ModeSymlink {
+		if err == ErrTraverseLink {
+			// Set callbackDone so we don't call it twice for both the
+			// symlink-as-symlink and the symlink-as-directory later:
+			w.enqueue(walkItem{dir: joined, callbackDone: true})
+			return nil
+		}
+		if err == filepath.SkipDir {
+			// Permit SkipDir on symlinks too.
+			return nil
+		}
+	}
+	return err
+}
+
+func (w *walker) walk(root string, runUserCallback bool) error {
+	if runUserCallback {
+		err := w.fn(root, os.ModeDir)
+		if err == filepath.SkipDir {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return readDir(root, w.onDirEnt)
+}

pkg/fastwalk/fastwalk_dirent_fileno.go

@@ -0,0 +1,13 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build freebsd openbsd netbsd
+
+package fastwalk
+
+import "syscall"
+
+func direntInode(dirent *syscall.Dirent) uint64 {
+	return uint64(dirent.Fileno)
+}

pkg/fastwalk/fastwalk_dirent_ino.go

@@ -0,0 +1,14 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin
+// +build !appengine
+
+package fastwalk
+
+import "syscall"
+
+func direntInode(dirent *syscall.Dirent) uint64 {
+	return uint64(dirent.Ino)
+}

pkg/fastwalk/fastwalk_dirent_namlen_bsd.go

@@ -0,0 +1,13 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd openbsd netbsd
+
+package fastwalk
+
+import "syscall"
+
+func direntNamlen(dirent *syscall.Dirent) uint64 {
+	return uint64(dirent.Namlen)
+}

pkg/fastwalk/fastwalk_dirent_namlen_linux.go

@@ -0,0 +1,29 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build !appengine
+
+package fastwalk
+
+import (
+	"bytes"
+	"syscall"
+	"unsafe"
+)
+
+func direntNamlen(dirent *syscall.Dirent) uint64 {
+	const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name))
+	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
+	const nameBufLen = uint16(len(nameBuf))
+	limit := dirent.Reclen - fixedHdr
+	if limit > nameBufLen {
+		limit = nameBufLen
+	}
+	nameLen := bytes.IndexByte(nameBuf[:limit], 0)
+	if nameLen < 0 {
+		panic("failed to find terminating 0 byte in dirent")
+	}
+	return uint64(nameLen)
+}

pkg/fastwalk/fastwalk_portable.go

@@ -0,0 +1,37 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd
+
+package fastwalk
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// readDir calls fn for each directory entry in dirName.
+// It does not descend into directories or follow symlinks.
+// If fn returns a non-nil error, readDir returns with that error
+// immediately.
+func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
+	fis, err := ioutil.ReadDir(dirName)
+	if err != nil {
+		return err
+	}
+	skipFiles := false
+	for _, fi := range fis {
+		if fi.Mode().IsRegular() && skipFiles {
+			continue
+		}
+		if err := fn(dirName, fi.Name(), fi.Mode()&os.ModeType); err != nil {
+			if err == ErrSkipFiles {
+				skipFiles = true
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}

pkg/fastwalk/fastwalk_test.go

@@ -0,0 +1,239 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fastwalk_test
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/photoprism/photoprism/pkg/fastwalk"
+)
+
+func formatFileModes(m map[string]os.FileMode) string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var buf bytes.Buffer
+	for _, k := range keys {
+		fmt.Fprintf(&buf, "%-20s: %v\n", k, m[k])
+	}
+	return buf.String()
+}
+
+func testFastWalk(t *testing.T, files map[string]string, callback func(path string, typ os.FileMode) error, want map[string]os.FileMode) {
+	tempdir, err := ioutil.TempDir("", "test-fast-walk")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempdir)
+	for path, contents := range files {
+		file := filepath.Join(tempdir, "/src", path)
+		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
+			t.Fatal(err)
+		}
+		var err error
+		if strings.HasPrefix(contents, "LINK:") {
+			err = os.Symlink(strings.TrimPrefix(contents, "LINK:"), file)
+		} else {
+			err = ioutil.WriteFile(file, []byte(contents), 0644)
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	got := map[string]os.FileMode{}
+	var mu sync.Mutex
+	if err := fastwalk.Walk(tempdir, func(path string, typ os.FileMode) error {
+		mu.Lock()
+		defer mu.Unlock()
+		if !strings.HasPrefix(path, tempdir) {
+			t.Fatalf("bogus prefix on %q, expect %q", path, tempdir)
+		}
+		key := filepath.ToSlash(strings.TrimPrefix(path, tempdir))
+		if old, dup := got[key]; dup {
+			t.Fatalf("callback called twice for key %q: %v -> %v", key, old, typ)
+		}
+		got[key] = typ
+		return callback(path, typ)
+	}); err != nil {
+		t.Fatalf("callback returned: %v", err)
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("walk mismatch.\n got:\n%v\nwant:\n%v", formatFileModes(got), formatFileModes(want))
+	}
+}
+
+func TestFastWalk_Basic(t *testing.T) {
+	testFastWalk(t, map[string]string{
+		"foo/foo.go":   "one",
+		"bar/bar.go":   "two",
+		"skip/skip.go": "skip",
+	},
+		func(path string, typ os.FileMode) error {
+			return nil
+		},
+		map[string]os.FileMode{
+			"":                  os.ModeDir,
+			"/src":              os.ModeDir,
+			"/src/bar":          os.ModeDir,
+			"/src/bar/bar.go":   0,
+			"/src/foo":          os.ModeDir,
+			"/src/foo/foo.go":   0,
+			"/src/skip":         os.ModeDir,
+			"/src/skip/skip.go": 0,
+		})
+}
+
+func TestFastWalk_LongFileName(t *testing.T) {
+	longFileName := strings.Repeat("x", 255)
+
+	testFastWalk(t, map[string]string{
+		longFileName: "one",
+	},
+		func(path string, typ os.FileMode) error {
+			return nil
+		},
+		map[string]os.FileMode{
+			"":                     os.ModeDir,
+			"/src":                 os.ModeDir,
+			"/src/" + longFileName: 0,
+		},
+	)
+}
+
+func TestFastWalk_Symlink(t *testing.T) {
+	switch runtime.GOOS {
+	case "windows", "plan9":
+		t.Skipf("skipping on %s", runtime.GOOS)
+	}
+	testFastWalk(t, map[string]string{
+		"foo/foo.go": "one",
+		"bar/bar.go": "LINK:../foo.go",
+		"symdir":     "LINK:foo",
+	},
+		func(path string, typ os.FileMode) error {
+			return nil
+		},
+		map[string]os.FileMode{
+			"":                os.ModeDir,
+			"/src":            os.ModeDir,
+			"/src/bar":        os.ModeDir,
+			"/src/bar/bar.go": os.ModeSymlink,
+			"/src/foo":        os.ModeDir,
+			"/src/foo/foo.go": 0,
+			"/src/symdir":     os.ModeSymlink,
+		})
+}
+
+func TestFastWalk_SkipDir(t *testing.T) {
+	testFastWalk(t, map[string]string{
+		"foo/foo.go":   "one",
+		"bar/bar.go":   "two",
+		"skip/skip.go": "skip",
+	},
+		func(path string, typ os.FileMode) error {
+			if typ == os.ModeDir && strings.HasSuffix(path, "skip") {
+				return filepath.SkipDir
+			}
+			return nil
+		},
+		map[string]os.FileMode{
+			"":                os.ModeDir,
+			"/src":            os.ModeDir,
+			"/src/bar":        os.ModeDir,
+			"/src/bar/bar.go": 0,
+			"/src/foo":        os.ModeDir,
+			"/src/foo/foo.go": 0,
+			"/src/skip":       os.ModeDir,
+		})
+}
+
+func TestFastWalk_SkipFiles(t *testing.T) {
+	// Directory iteration order is undefined, so there's no way to know
+	// which file to expect until the walk happens. Rather than mess
+	// with the test infrastructure, just mutate want.
+	var mu sync.Mutex
+	want := map[string]os.FileMode{
+		"":              os.ModeDir,
+		"/src":          os.ModeDir,
+		"/src/zzz":      os.ModeDir,
+		"/src/zzz/c.go": 0,
+	}
+
+	testFastWalk(t, map[string]string{
+		"a_skipfiles.go": "a",
+		"b_skipfiles.go": "b",
+		"zzz/c.go":       "c",
+	},
+		func(path string, typ os.FileMode) error {
+			if strings.HasSuffix(path, "_skipfiles.go") {
+				mu.Lock()
+				defer mu.Unlock()
+				want["/src/"+filepath.Base(path)] = 0
+				return fastwalk.ErrSkipFiles
+			}
+			return nil
+		},
+		want)
+	if len(want) != 5 {
+		t.Errorf("saw too many files: wanted 5, got %v (%v)", len(want), want)
+	}
+}
+
+func TestFastWalk_TraverseSymlink(t *testing.T) {
+	switch runtime.GOOS {
+	case "windows", "plan9":
+		t.Skipf("skipping on %s", runtime.GOOS)
+	}
+
+	testFastWalk(t, map[string]string{
+		"foo/foo.go":   "one",
+		"bar/bar.go":   "two",
+		"skip/skip.go": "skip",
+		"symdir":       "LINK:foo",
+	},
+		func(path string, typ os.FileMode) error {
+			if typ == os.ModeSymlink {
+				return fastwalk.ErrTraverseLink
+			}
+			return nil
+		},
+		map[string]os.FileMode{
+			"":                   os.ModeDir,
+			"/src":               os.ModeDir,
+			"/src/bar":           os.ModeDir,
+			"/src/bar/bar.go":    0,
+			"/src/foo":           os.ModeDir,
+			"/src/foo/foo.go":    0,
+			"/src/skip":          os.ModeDir,
+			"/src/skip/skip.go":  0,
+			"/src/symdir":        os.ModeSymlink,
+			"/src/symdir/foo.go": 0,
+		})
+}
+
+var benchDir = flag.String("benchdir", runtime.GOROOT(), "The directory to scan for BenchmarkFastWalk")
+
+func BenchmarkFastWalk(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		err := fastwalk.Walk(*benchDir, func(path string, typ os.FileMode) error { return nil })
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/fastwalk/fastwalk_unix.go

@@ -0,0 +1,128 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin freebsd openbsd netbsd
+// +build !appengine
+
+package fastwalk
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+const blockSize = 8 << 10
+
+// unknownFileMode is a sentinel (and bogus) os.FileMode
+// value used to represent a syscall.DT_UNKNOWN Dirent.Type.
+const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice
+
+func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
+	fd, err := syscall.Open(dirName, 0, 0)
+	if err != nil {
+		return &os.PathError{Op: "open", Path: dirName, Err: err}
+	}
+	defer syscall.Close(fd)
+
+	// The buffer must be at least a block long.
+	buf := make([]byte, blockSize) // stack-allocated; doesn't escape
+	bufp := 0                      // starting read position in buf
+	nbuf := 0                      // end valid data in buf
+	skipFiles := false
+	for {
+		if bufp >= nbuf {
+			bufp = 0
+			nbuf, err = syscall.ReadDirent(fd, buf)
+			if err != nil {
+				return os.NewSyscallError("readdirent", err)
+			}
+			if nbuf <= 0 {
+				return nil
+			}
+		}
+		consumed, name, typ := parseDirEnt(buf[bufp:nbuf])
+		bufp += consumed
+		if name == "" || name == "." || name == ".." {
+			continue
+		}
+		// Fallback for filesystems (like old XFS) that don't
+		// support Dirent.Type and have DT_UNKNOWN (0) there
+		// instead.
+		if typ == unknownFileMode {
+			fi, err := os.Lstat(dirName + "/" + name)
+			if err != nil {
+				// It got deleted in the meantime.
+				if os.IsNotExist(err) {
+					continue
+				}
+				return err
+			}
+			typ = fi.Mode() & os.ModeType
+		}
+		if skipFiles && typ.IsRegular() {
+			continue
+		}
+		if err := fn(dirName, name, typ); err != nil {
+			if err == ErrSkipFiles {
+				skipFiles = true
+				continue
+			}
+			return err
+		}
+	}
+}
+
+func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) {
+	// golang.org/issue/37269
+	dirent := &syscall.Dirent{}
+	copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf)
+	if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
+		panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v))
+	}
+	if len(buf) < int(dirent.Reclen) {
+		panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen))
+	}
+	consumed = int(dirent.Reclen)
+	if direntInode(dirent) == 0 { // File absent in directory.
+		return
+	}
+	switch dirent.Type {
+	case syscall.DT_REG:
+		typ = 0
+	case syscall.DT_DIR:
+		typ = os.ModeDir
+	case syscall.DT_LNK:
+		typ = os.ModeSymlink
+	case syscall.DT_BLK:
+		typ = os.ModeDevice
+	case syscall.DT_FIFO:
+		typ = os.ModeNamedPipe
+	case syscall.DT_SOCK:
+		typ = os.ModeSocket
+	case syscall.DT_UNKNOWN:
+		typ = unknownFileMode
+	default:
+		// Skip weird things.
+		// It's probably a DT_WHT (http://lwn.net/Articles/325369/)
+		// or something. Revisit if/when this package is moved outside
+		// of goimports. goimports only cares about regular files,
+		// symlinks, and directories.
+		return
+	}
+
+	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
+	nameLen := direntNamlen(dirent)
+
+	// Special cases for common things:
+	if nameLen == 1 && nameBuf[0] == '.' {
+		name = "."
+	} else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' {
+		name = ".."
+	} else {
+		name = string(nameBuf[:nameLen])
+	}
+	return
+}