Implemented index command

Dockerfile

@@ -82,7 +82,7 @@ RUN mkdir -m 777 /go/pkg/dep
 # USER photoprism
 
 # Set up project directory
-WORKDIR "/go/src/photoprism"
+WORKDIR "/go/src/github.com/photoprism/photoprism"
 COPY . .
 
 RUN dep ensure

Gopkg.lock

@@ -4,14 +4,14 @@
 [[projects]]
   name = "cloud.google.com/go"
   packages = ["civil"]
-  revision = "777200caa7fb8936aed0f12b1fd79af64cc83ec9"
-  version = "v0.24.0"
+  revision = "aad3f485ee528456e0768f20397b4d9dd941e755"
+  version = "v0.25.0"
 
 [[projects]]
   branch = "master"
   name = "github.com/araddon/dateparse"
   packages = ["."]
-  revision = "6135c1994ea28aa1bb341f5c704885906f584904"
+  revision = "089f77b1d92b615cc77fde0d2fa528b5e85e832d"
 
 [[projects]]
   branch = "master"
@@ -38,13 +38,13 @@
     ".",
     "internal/cp"
   ]
-  revision = "94c9c97e8c9f9844d15c846854a7a6031ae2132c"
+  revision = "242fa5aa1b45aeb9fcdfeee88822982e3f548e22"
 
 [[projects]]
   name = "github.com/disintegration/imaging"
   packages = ["."]
-  revision = "1884593a19ddc6f2ea050403430d02c1d0fc1283"
-  version = "v1.3.0"
+  revision = "bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134"
+  version = "v1.4.2"
 
 [[projects]]
   name = "github.com/djherbis/times"
@@ -98,12 +98,6 @@
   revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4"
   version = "v1.9.0"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/photoprism/photoprism"
-  packages = ["."]
-  revision = "b2659ba5ce48b223490b8f51db065d93ae8f0cf5"
-
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -124,7 +118,7 @@
     "mknote",
     "tiff"
   ]
-  revision = "17202558c8d9c3fd047859f1a5e73fd9ae709187"
+  revision = "8d986c03457a2057c7b0fb0a48113f7dd48f9619"
 
 [[projects]]
   name = "github.com/steakknife/hamming"
@@ -135,8 +129,8 @@
 [[projects]]
   name = "github.com/stretchr/testify"
   packages = ["assert"]
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
 
 [[projects]]
   name = "github.com/tensorflow/tensorflow"
@@ -144,8 +138,8 @@
     "tensorflow/go",
     "tensorflow/go/op"
   ]
-  revision = "37aa430d84ced579342a4044c89c236664be7f68"
-  version = "v1.5.0"
+  revision = "25c197e02393bd44f50079945409009dd4d434f8"
+  version = "v1.9.0"
 
 [[projects]]
   name = "github.com/urfave/cli"
@@ -171,7 +165,7 @@
     "vp8l",
     "webp"
   ]
-  revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
+  revision = "c73c2afc3b812cdd6385de5a50616511c4a3d458"
 
 [[projects]]
   name = "google.golang.org/appengine"
@@ -182,6 +176,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "8aa59b793f2c56ca48723acc6a2b517cbd2dd323af673b71c1bcc74d491951bf"
+  inputs-digest = "a6f9a83ff2c8ea983a59f7664e5594f0b44e75bf123a6e5b4643574e5ec1765f"
   solver-name = "gps-cdcl"
   solver-version = 1

cmd/photoprism/photoprism.go

@@ -14,7 +14,7 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "PhotoPrism"
 	app.Usage = "Digital Photo Archive"
-	app.Version = "0.1.0"
+	app.Version = "0.2.0"
 	app.Flags = globalCliFlags
 	app.Commands = []cli.Command{
 		{
@@ -65,7 +65,9 @@ func main() {
 
 				fmt.Printf("Importing photos from %s...\n", conf.ImportPath)
 
-				importer := photoprism.NewImporter(conf.OriginalsPath, conf.GetDb())
+				indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+				importer := photoprism.NewImporter(conf.OriginalsPath, indexer)
 
 				importer.ImportPhotosFromDirectory(conf.ImportPath)
 
@@ -74,6 +76,27 @@ func main() {
 				return nil
 			},
 		},
+		{
+			Name:  "index",
+			Usage: "Re-indexes all originals",
+			Action: func(context *cli.Context) error {
+				conf.SetValuesFromFile(photoprism.GetExpandedFilename(context.GlobalString("config-file")))
+
+				conf.SetValuesFromCliContext(context)
+
+				conf.CreateDirectories()
+
+				fmt.Printf("Indexing photos in %s...\n", conf.OriginalsPath)
+
+				indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+				indexer.IndexAll()
+
+				fmt.Println("Done.")
+
+				return nil
+			},
+		},
 		{
 			Name:  "convert",
 			Usage: "Converts RAW originals to JPEG",

config.example.yml

@@ -4,4 +4,4 @@ thumbnails-path: photos/thumbnails
 import-path: photos/import
 export-path: photos/export
 database-driver: mysql
-database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism
+database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true

config_test.go

@@ -19,7 +19,7 @@ var thumbnailsPath = GetExpandedFilename(testDataPath + "/thumbnails")
 var importPath = GetExpandedFilename(testDataPath + "/import")
 var exportPath = GetExpandedFilename(testDataPath + "/export")
 var databaseDriver = "mysql"
-var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism"
+var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true"
 
 func (c *Config) RemoveTestData(t *testing.T) {
 	os.RemoveAll(c.ImportPath)

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       - 80:80
       - 8080:8080
     volumes:
-      - .:/go/src/photoprism
+      - .:/go/src/github.com/photoprism/photoprism
 
   database:
     image: mysql:latest

importer.go

@@ -2,7 +2,6 @@ package photoprism
 
 import (
 	"fmt"
-	"github.com/jinzhu/gorm"
 	"github.com/pkg/errors"
 	"log"
 	"os"
@@ -14,16 +13,16 @@ import (
 
 type Importer struct {
 	originalsPath          string
-	db                     *gorm.DB
+	indexer                *Indexer
 	removeDotFiles         bool
 	removeExistingFiles    bool
 	removeEmptyDirectories bool
 }
 
-func NewImporter(originalsPath string, db *gorm.DB) *Importer {
+func NewImporter(originalsPath string, indexer *Indexer) *Importer {
 	instance := &Importer{
 		originalsPath:          originalsPath,
-		db:                     db,
+		indexer:                indexer,
 		removeDotFiles:         true,
 		removeExistingFiles:    true,
 		removeEmptyDirectories: true,
@@ -66,14 +65,13 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
 				os.MkdirAll(path.Dir(destinationFilename), os.ModePerm)
 				log.Printf("Moving file %s to %s", relatedMediaFile.GetFilename(), destinationFilename)
 				relatedMediaFile.Move(destinationFilename)
+				i.indexer.IndexMediaFile(relatedMediaFile)
 			} else if i.removeExistingFiles {
 				relatedMediaFile.Remove()
 				log.Printf("Deleted %s (already exists)", relatedMediaFile.GetFilename())
 			}
 		}
 
-		// mediaFile.Move(i.originalsPath)
-
 		return nil
 	})
 
@@ -115,7 +113,7 @@ func (i *Importer) GetDestinationFilename(masterFile *MediaFile, mediaFile *Medi
 
 		iteration++
 
-		result = pathName + "/" + canonicalName + "_" + fmt.Sprintf("V%d", iteration) + fileExtension
+		result = pathName + "/" + canonicalName + "." + fmt.Sprintf("V%d", iteration) + fileExtension
 	}
 
 	return result, nil

importer_test.go

@@ -8,7 +8,9 @@ import (
 func TestNewImporter(t *testing.T) {
 	conf := NewTestConfig()
 
-	importer := NewImporter(conf.OriginalsPath)
+	indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+	importer := NewImporter(conf.OriginalsPath, indexer)
 
 	assert.IsType(t, &Importer{}, importer)
 }
@@ -18,7 +20,9 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) {
 
 	conf.InitializeTestData(t)
 
-	importer := NewImporter(conf.OriginalsPath)
+	indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+	importer := NewImporter(conf.OriginalsPath, indexer)
 
 	importer.ImportPhotosFromDirectory(conf.ImportPath)
 }
@@ -26,7 +30,10 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) {
 func TestImporter_GetDestinationFilename(t *testing.T) {
 	conf := NewTestConfig()
 	conf.InitializeTestData(t)
-	importer := NewImporter(conf.OriginalsPath)
+
+	indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+	importer := NewImporter(conf.OriginalsPath, indexer)
 
 	rawFile := NewMediaFile(conf.ImportPath + "/raw/IMG_1435.CR2")
 
@@ -34,5 +41,5 @@ func TestImporter_GetDestinationFilename(t *testing.T) {
 
 	assert.Empty(t, err)
 
-	assert.Equal(t, conf.OriginalsPath+"/2018/02/20180204_180813_B0770443A5F7.cr2", filename)
+	assert.Equal(t, conf.OriginalsPath + "/2018/02/20180204_170813_B0770443A5F7.cr2", filename)
 }

indexer.go

@@ -1 +1,119 @@
 package photoprism
+
+import (
+	"github.com/jinzhu/gorm"
+	"os"
+	"strings"
+	"path/filepath"
+	"log"
+	"io/ioutil"
+	"github.com/photoprism/photoprism/recognize"
+)
+
+type Indexer struct {
+	originalsPath string
+	db            *gorm.DB
+}
+
+func NewIndexer(originalsPath string, db *gorm.DB) *Indexer {
+	instance := &Indexer{
+		originalsPath: originalsPath,
+		db:            db,
+	}
+
+	return instance
+}
+
+func (i *Indexer) GetImageTags(jpeg *MediaFile) (result []Tag) {
+	if imageBuffer, err := ioutil.ReadFile(jpeg.filename); err == nil {
+		tags, err := recognize.GetImageTags(string(imageBuffer))
+
+		if err != nil {
+			return result
+		}
+
+		for _, tag := range tags {
+			if tag.Probability > 0.2 {
+				result = append(result, Tag{Label: tag.Label})
+			}
+		}
+	}
+
+	return result
+}
+
+func (i *Indexer) IndexMediaFile(mediaFile *MediaFile) {
+	var photo Photo
+	var file File
+
+	canonicalName := mediaFile.GetCanonicalNameFromFile()
+	fileHash := mediaFile.GetHash()
+
+	if result := i.db.First(&photo, "canonical_name = ?", canonicalName); result.Error != nil {
+		if jpeg, err := mediaFile.GetJpeg(); err == nil {
+			if perceptualHash, err := jpeg.GetPerceptualHash(); err == nil {
+				photo.PerceptualHash = perceptualHash
+			}
+
+			if exifData, err := jpeg.GetExifData(); err == nil {
+				photo.Lat = exifData.Lat
+				photo.Long = exifData.Long
+			}
+
+			photo.Tags = i.GetImageTags(jpeg)
+		}
+
+		photo.CanonicalName = canonicalName
+		photo.Files = []File{}
+		photo.Albums = []Album{}
+		photo.Author = ""
+		photo.CameraModel = mediaFile.GetCameraModel()
+		photo.LocationName = ""
+		photo.Liked = false
+		photo.Private = true
+		photo.Deleted = false
+
+		i.db.Create(&photo)
+	}
+
+	if result := i.db.First(&file, "hash = ?", fileHash); result.Error != nil {
+		file.PhotoID = photo.ID
+		file.Filename = mediaFile.GetFilename()
+		file.Hash = fileHash
+		file.FileType = mediaFile.GetType()
+		file.MimeType = mediaFile.GetMimeType()
+
+		i.db.Create(&file)
+	}
+}
+
+func (i *Indexer) IndexAll() {
+	err := filepath.Walk(i.originalsPath, func(filename string, fileInfo os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+
+		if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
+			return nil
+		}
+
+		mediaFile := NewMediaFile(filename)
+
+		if !mediaFile.Exists() || !mediaFile.IsPhoto() {
+			return nil
+		}
+
+		relatedFiles, _, _ := mediaFile.GetRelatedFiles()
+
+		for _, relatedMediaFile := range relatedFiles {
+			log.Printf("Indexing %s", relatedMediaFile.GetFilename())
+			i.IndexMediaFile(relatedMediaFile)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		log.Print(err.Error())
+	}
+}

mediafile.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -38,6 +39,7 @@ var FileExtensions = map[string]string{
 	".avi":  FileTypeMovie,
 	".yml":  FileTypeYaml,
 	".jpg":  FileTypeJpeg,
+	".thm":  FileTypeJpeg,
 	".jpeg": FileTypeJpeg,
 	".xmp":  FileTypeXmp,
 	".aae":  FileTypeAae,
@@ -110,6 +112,16 @@ func (m *MediaFile) GetCanonicalName() string {
 	return result
 }
 
+func (m *MediaFile) GetCanonicalNameFromFile() string {
+	basename := filepath.Base(m.GetFilename())
+
+	if end := strings.Index(basename, "."); end != -1 {
+		return basename[:end] // Length of canonical name: 16 + 12
+	} else {
+		return basename
+	}
+}
+
 func (m *MediaFile) GetPerceptualHash() (string, error) {
 	if m.perceptualHash != "" {
 		return m.perceptualHash, nil
@@ -297,12 +309,16 @@ func (m *MediaFile) IsJpeg() bool {
 	return m.GetMimeType() == MimeTypeJpeg
 }
 
+func (m *MediaFile) GetType() string {
+	return FileExtensions[m.GetExtension()]
+}
+
 func (m *MediaFile) HasType(typeString string) bool {
 	if typeString == FileTypeJpeg {
 		return m.IsJpeg()
 	}
 
-	return FileExtensions[m.GetExtension()] == typeString
+	return m.GetType() == typeString
 }
 
 func (m *MediaFile) IsRaw() bool {
@@ -312,3 +328,19 @@ func (m *MediaFile) IsRaw() bool {
 func (m *MediaFile) IsPhoto() bool {
 	return m.IsJpeg() || m.IsRaw()
 }
+
+func (m *MediaFile) GetJpeg() (*MediaFile, error) {
+	if m.IsJpeg() {
+		return m, nil
+	}
+
+	jpegFilename := m.GetFilename()[0:len(m.GetFilename()) - len(filepath.Ext(m.GetFilename()))] + ".jpg"
+
+	if !fileExists(jpegFilename) {
+		return nil, errors.New("file does not exist")
+	}
+
+	result := NewMediaFile(jpegFilename)
+
+	return result, nil
+}

recognize/image.go

@@ -1,4 +1,4 @@
-package tensorflow
+package recognize
 
 import (
 	tf "github.com/tensorflow/tensorflow/tensorflow/go"

recognize/recognize.go

@@ -1,4 +1,4 @@
-package tensorflow
+package recognize
 
 import (
 	"bufio"
@@ -25,7 +25,7 @@ var (
 	labels []string
 )
 
-func RecognizeImage(image string) (result []LabelResult, err error) {
+func GetImageTags(image string) (result []LabelResult, err error) {
 	if err := loadModel(); err != nil {
 		return nil, err
 	}

recognize/recognize_test.go

@@ -1,4 +1,4 @@
-package tensorflow
+package recognize
 
 import (
 	"testing"
@@ -6,11 +6,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestRecognizeImage(t *testing.T) {
+func TestGetImageTags(t *testing.T) {
 	if imageBuffer, err := ioutil.ReadFile("cat.jpg"); err != nil {
 		t.Error(err)
 	} else {
-		result, err := RecognizeImage(string(imageBuffer))
+		result, err := GetImageTags(string(imageBuffer))
 
 		assert.NotNil(t, result)
 		assert.Nil(t, err)

thumbnails_test.go

@@ -44,7 +44,9 @@ func TestCreateThumbnailsFromOriginals(t *testing.T) {
 
 	conf.InitializeTestData(t)
 
-	importer := NewImporter(conf.OriginalsPath)
+	indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
+
+	importer := NewImporter(conf.OriginalsPath, indexer)
 
 	importer.ImportPhotosFromDirectory(conf.ImportPath)