diff --git a/Dockerfile b/Dockerfile index 3cfa283c5..97b491636 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ RUN mkdir -m 777 /go/pkg/dep # USER photoprism # Set up project directory -WORKDIR "/go/src/photoprism" +WORKDIR "/go/src/github.com/photoprism/photoprism" COPY . . RUN dep ensure diff --git a/Gopkg.lock b/Gopkg.lock index 4fb6ce406..23efcc32e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,14 +4,14 @@ [[projects]] name = "cloud.google.com/go" packages = ["civil"] - revision = "777200caa7fb8936aed0f12b1fd79af64cc83ec9" - version = "v0.24.0" + revision = "aad3f485ee528456e0768f20397b4d9dd941e755" + version = "v0.25.0" [[projects]] branch = "master" name = "github.com/araddon/dateparse" packages = ["."] - revision = "6135c1994ea28aa1bb341f5c704885906f584904" + revision = "089f77b1d92b615cc77fde0d2fa528b5e85e832d" [[projects]] branch = "master" @@ -38,13 +38,13 @@ ".", "internal/cp" ] - revision = "94c9c97e8c9f9844d15c846854a7a6031ae2132c" + revision = "242fa5aa1b45aeb9fcdfeee88822982e3f548e22" [[projects]] name = "github.com/disintegration/imaging" packages = ["."] - revision = "1884593a19ddc6f2ea050403430d02c1d0fc1283" - version = "v1.3.0" + revision = "bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134" + version = "v1.4.2" [[projects]] name = "github.com/djherbis/times" @@ -98,12 +98,6 @@ revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4" version = "v1.9.0" -[[projects]] - branch = "master" - name = "github.com/photoprism/photoprism" - packages = ["."] - revision = "b2659ba5ce48b223490b8f51db065d93ae8f0cf5" - [[projects]] name = "github.com/pkg/errors" packages = ["."] @@ -124,7 +118,7 @@ "mknote", "tiff" ] - revision = "17202558c8d9c3fd047859f1a5e73fd9ae709187" + revision = "8d986c03457a2057c7b0fb0a48113f7dd48f9619" [[projects]] name = "github.com/steakknife/hamming" @@ -135,8 +129,8 @@ [[projects]] name = "github.com/stretchr/testify" packages = ["assert"] - revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" - version = "v1.2.1" + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" [[projects]] name = "github.com/tensorflow/tensorflow" @@ -144,8 +138,8 @@ "tensorflow/go", "tensorflow/go/op" ] - revision = "37aa430d84ced579342a4044c89c236664be7f68" - version = "v1.5.0" + revision = "25c197e02393bd44f50079945409009dd4d434f8" + version = "v1.9.0" [[projects]] name = "github.com/urfave/cli" @@ -171,7 +165,7 @@ "vp8l", "webp" ] - revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7" + revision = "c73c2afc3b812cdd6385de5a50616511c4a3d458" [[projects]] name = "google.golang.org/appengine" @@ -182,6 +176,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8aa59b793f2c56ca48723acc6a2b517cbd2dd323af673b71c1bcc74d491951bf" + inputs-digest = "a6f9a83ff2c8ea983a59f7664e5594f0b44e75bf123a6e5b4643574e5ec1765f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/photoprism/photoprism.go b/cmd/photoprism/photoprism.go index c33b052f2..e55ee60ff 100644 --- a/cmd/photoprism/photoprism.go +++ b/cmd/photoprism/photoprism.go @@ -14,7 +14,7 @@ func main() { app := cli.NewApp() app.Name = "PhotoPrism" app.Usage = "Digital Photo Archive" - app.Version = "0.1.0" + app.Version = "0.2.0" app.Flags = globalCliFlags app.Commands = []cli.Command{ { @@ -65,7 +65,9 @@ func main() { fmt.Printf("Importing photos from %s...\n", conf.ImportPath) - importer := photoprism.NewImporter(conf.OriginalsPath, conf.GetDb()) + indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb()) + + importer := photoprism.NewImporter(conf.OriginalsPath, indexer) importer.ImportPhotosFromDirectory(conf.ImportPath) @@ -74,6 +76,27 @@ func main() { return nil }, }, + { + Name: "index", + Usage: "Re-indexes all originals", + Action: func(context *cli.Context) error { + conf.SetValuesFromFile(photoprism.GetExpandedFilename(context.GlobalString("config-file"))) + + conf.SetValuesFromCliContext(context) + + conf.CreateDirectories() + + fmt.Printf("Indexing photos in %s...\n", conf.OriginalsPath) + + indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb()) + + indexer.IndexAll() + + fmt.Println("Done.") + + return nil + }, + }, { Name: "convert", Usage: "Converts RAW originals to JPEG", diff --git a/config.example.yml b/config.example.yml index 49ba83fe9..582a89aa0 100644 --- a/config.example.yml +++ b/config.example.yml @@ -4,4 +4,4 @@ thumbnails-path: photos/thumbnails import-path: photos/import export-path: photos/export database-driver: mysql -database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism \ No newline at end of file +database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true \ No newline at end of file diff --git a/config_test.go b/config_test.go index a2cbab0bd..80e45bcce 100644 --- a/config_test.go +++ b/config_test.go @@ -19,7 +19,7 @@ var thumbnailsPath = GetExpandedFilename(testDataPath + "/thumbnails") var importPath = GetExpandedFilename(testDataPath + "/import") var exportPath = GetExpandedFilename(testDataPath + "/export") var databaseDriver = "mysql" -var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism" +var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true" func (c *Config) RemoveTestData(t *testing.T) { os.RemoveAll(c.ImportPath) diff --git a/docker-compose.yml b/docker-compose.yml index 21a690827..d25004383 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - 80:80 - 8080:8080 volumes: - - .:/go/src/photoprism + - .:/go/src/github.com/photoprism/photoprism database: image: mysql:latest diff --git a/importer.go b/importer.go index f19ce9ab1..703e54b8c 100644 --- a/importer.go +++ b/importer.go @@ -2,7 +2,6 @@ package photoprism import ( "fmt" - "github.com/jinzhu/gorm" "github.com/pkg/errors" "log" "os" @@ -14,16 +13,16 @@ import ( type Importer struct { originalsPath string - db *gorm.DB + indexer *Indexer removeDotFiles bool removeExistingFiles bool removeEmptyDirectories bool } -func NewImporter(originalsPath string, db *gorm.DB) *Importer { +func NewImporter(originalsPath string, indexer *Indexer) *Importer { instance := &Importer{ originalsPath: originalsPath, - db: db, + indexer: indexer, removeDotFiles: true, removeExistingFiles: true, removeEmptyDirectories: true, @@ -66,14 +65,13 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { os.MkdirAll(path.Dir(destinationFilename), os.ModePerm) log.Printf("Moving file %s to %s", relatedMediaFile.GetFilename(), destinationFilename) relatedMediaFile.Move(destinationFilename) + i.indexer.IndexMediaFile(relatedMediaFile) } else if i.removeExistingFiles { relatedMediaFile.Remove() log.Printf("Deleted %s (already exists)", relatedMediaFile.GetFilename()) } } - // mediaFile.Move(i.originalsPath) - return nil }) @@ -115,7 +113,7 @@ func (i *Importer) GetDestinationFilename(masterFile *MediaFile, mediaFile *Medi iteration++ - result = pathName + "/" + canonicalName + "_" + fmt.Sprintf("V%d", iteration) + fileExtension + result = pathName + "/" + canonicalName + "." + fmt.Sprintf("V%d", iteration) + fileExtension } return result, nil diff --git a/importer_test.go b/importer_test.go index b849da12c..7082e7957 100644 --- a/importer_test.go +++ b/importer_test.go @@ -8,7 +8,9 @@ import ( func TestNewImporter(t *testing.T) { conf := NewTestConfig() - importer := NewImporter(conf.OriginalsPath) + indexer := NewIndexer(conf.OriginalsPath, conf.GetDb()) + + importer := NewImporter(conf.OriginalsPath, indexer) assert.IsType(t, &Importer{}, importer) } @@ -18,7 +20,9 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) { conf.InitializeTestData(t) - importer := NewImporter(conf.OriginalsPath) + indexer := NewIndexer(conf.OriginalsPath, conf.GetDb()) + + importer := NewImporter(conf.OriginalsPath, indexer) importer.ImportPhotosFromDirectory(conf.ImportPath) } @@ -26,7 +30,10 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) { func TestImporter_GetDestinationFilename(t *testing.T) { conf := NewTestConfig() conf.InitializeTestData(t) - importer := NewImporter(conf.OriginalsPath) + + indexer := NewIndexer(conf.OriginalsPath, conf.GetDb()) + + importer := NewImporter(conf.OriginalsPath, indexer) rawFile := NewMediaFile(conf.ImportPath + "/raw/IMG_1435.CR2") @@ -34,5 +41,5 @@ func TestImporter_GetDestinationFilename(t *testing.T) { assert.Empty(t, err) - assert.Equal(t, conf.OriginalsPath+"/2018/02/20180204_180813_B0770443A5F7.cr2", filename) + assert.Equal(t, conf.OriginalsPath + "/2018/02/20180204_170813_B0770443A5F7.cr2", filename) } diff --git a/indexer.go b/indexer.go index 06a037bb5..911ba56fb 100644 --- a/indexer.go +++ b/indexer.go @@ -1 +1,119 @@ package photoprism + +import ( + "github.com/jinzhu/gorm" + "os" + "strings" + "path/filepath" + "log" + "io/ioutil" + "github.com/photoprism/photoprism/recognize" +) + +type Indexer struct { + originalsPath string + db *gorm.DB +} + +func NewIndexer(originalsPath string, db *gorm.DB) *Indexer { + instance := &Indexer{ + originalsPath: originalsPath, + db: db, + } + + return instance +} + +func (i *Indexer) GetImageTags(jpeg *MediaFile) (result []Tag) { + if imageBuffer, err := ioutil.ReadFile(jpeg.filename); err == nil { + tags, err := recognize.GetImageTags(string(imageBuffer)) + + if err != nil { + return result + } + + for _, tag := range tags { + if tag.Probability > 0.2 { + result = append(result, Tag{Label: tag.Label}) + } + } + } + + return result +} + +func (i *Indexer) IndexMediaFile(mediaFile *MediaFile) { + var photo Photo + var file File + + canonicalName := mediaFile.GetCanonicalNameFromFile() + fileHash := mediaFile.GetHash() + + if result := i.db.First(&photo, "canonical_name = ?", canonicalName); result.Error != nil { + if jpeg, err := mediaFile.GetJpeg(); err == nil { + if perceptualHash, err := jpeg.GetPerceptualHash(); err == nil { + photo.PerceptualHash = perceptualHash + } + + if exifData, err := jpeg.GetExifData(); err == nil { + photo.Lat = exifData.Lat + photo.Long = exifData.Long + } + + photo.Tags = i.GetImageTags(jpeg) + } + + photo.CanonicalName = canonicalName + photo.Files = []File{} + photo.Albums = []Album{} + photo.Author = "" + photo.CameraModel = mediaFile.GetCameraModel() + photo.LocationName = "" + photo.Liked = false + photo.Private = true + photo.Deleted = false + + i.db.Create(&photo) + } + + if result := i.db.First(&file, "hash = ?", fileHash); result.Error != nil { + file.PhotoID = photo.ID + file.Filename = mediaFile.GetFilename() + file.Hash = fileHash + file.FileType = mediaFile.GetType() + file.MimeType = mediaFile.GetMimeType() + + i.db.Create(&file) + } +} + +func (i *Indexer) IndexAll() { + err := filepath.Walk(i.originalsPath, func(filename string, fileInfo os.FileInfo, err error) error { + if err != nil { + return nil + } + + if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") { + return nil + } + + mediaFile := NewMediaFile(filename) + + if !mediaFile.Exists() || !mediaFile.IsPhoto() { + return nil + } + + relatedFiles, _, _ := mediaFile.GetRelatedFiles() + + for _, relatedMediaFile := range relatedFiles { + log.Printf("Indexing %s", relatedMediaFile.GetFilename()) + i.IndexMediaFile(relatedMediaFile) + } + + return nil + }) + + if err != nil { + log.Print(err.Error()) + } +} diff --git a/mediafile.go b/mediafile.go index 53575247a..7f107fe5b 100644 --- a/mediafile.go +++ b/mediafile.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" "time" + "github.com/pkg/errors" ) const ( @@ -38,6 +39,7 @@ var FileExtensions = map[string]string{ ".avi": FileTypeMovie, ".yml": FileTypeYaml, ".jpg": FileTypeJpeg, + ".thm": FileTypeJpeg, ".jpeg": FileTypeJpeg, ".xmp": FileTypeXmp, ".aae": FileTypeAae, @@ -110,6 +112,16 @@ func (m *MediaFile) GetCanonicalName() string { return result } +func (m *MediaFile) GetCanonicalNameFromFile() string { + basename := filepath.Base(m.GetFilename()) + + if end := strings.Index(basename, "."); end != -1 { + return basename[:end] // Length of canonical name: 16 + 12 + } else { + return basename + } +} + func (m *MediaFile) GetPerceptualHash() (string, error) { if m.perceptualHash != "" { return m.perceptualHash, nil @@ -297,12 +309,16 @@ func (m *MediaFile) IsJpeg() bool { return m.GetMimeType() == MimeTypeJpeg } +func (m *MediaFile) GetType() string { + return FileExtensions[m.GetExtension()] +} + func (m *MediaFile) HasType(typeString string) bool { if typeString == FileTypeJpeg { return m.IsJpeg() } - return FileExtensions[m.GetExtension()] == typeString + return m.GetType() == typeString } func (m *MediaFile) IsRaw() bool { @@ -312,3 +328,19 @@ func (m *MediaFile) IsRaw() bool { func (m *MediaFile) IsPhoto() bool { return m.IsJpeg() || m.IsRaw() } + +func (m *MediaFile) GetJpeg() (*MediaFile, error) { + if m.IsJpeg() { + return m, nil + } + + jpegFilename := m.GetFilename()[0:len(m.GetFilename()) - len(filepath.Ext(m.GetFilename()))] + ".jpg" + + if !fileExists(jpegFilename) { + return nil, errors.New("file does not exist") + } + + result := NewMediaFile(jpegFilename) + + return result, nil +} diff --git a/tensorflow/cat.jpg b/recognize/cat.jpg similarity index 100% rename from tensorflow/cat.jpg rename to recognize/cat.jpg diff --git a/tensorflow/image.go b/recognize/image.go similarity index 98% rename from tensorflow/image.go rename to recognize/image.go index 1c9a7e681..964b9331a 100644 --- a/tensorflow/image.go +++ b/recognize/image.go @@ -1,4 +1,4 @@ -package tensorflow +package recognize import ( tf "github.com/tensorflow/tensorflow/tensorflow/go" diff --git a/tensorflow/recognize.go b/recognize/recognize.go similarity index 96% rename from tensorflow/recognize.go rename to recognize/recognize.go index 2ed56e46b..8ff4d3378 100644 --- a/tensorflow/recognize.go +++ b/recognize/recognize.go @@ -1,4 +1,4 @@ -package tensorflow +package recognize import ( "bufio" @@ -25,7 +25,7 @@ var ( labels []string ) -func RecognizeImage(image string) (result []LabelResult, err error) { +func GetImageTags(image string) (result []LabelResult, err error) { if err := loadModel(); err != nil { return nil, err } diff --git a/tensorflow/recognize_test.go b/recognize/recognize_test.go similarity index 79% rename from tensorflow/recognize_test.go rename to recognize/recognize_test.go index ce03e8fd1..d594e05d6 100644 --- a/tensorflow/recognize_test.go +++ b/recognize/recognize_test.go @@ -1,4 +1,4 @@ -package tensorflow +package recognize import ( "testing" @@ -6,11 +6,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRecognizeImage(t *testing.T) { +func TestGetImageTags(t *testing.T) { if imageBuffer, err := ioutil.ReadFile("cat.jpg"); err != nil { t.Error(err) } else { - result, err := RecognizeImage(string(imageBuffer)) + result, err := GetImageTags(string(imageBuffer)) assert.NotNil(t, result) assert.Nil(t, err) diff --git a/thumbnails_test.go b/thumbnails_test.go index a292db6bf..21c1647ba 100644 --- a/thumbnails_test.go +++ b/thumbnails_test.go @@ -44,7 +44,9 @@ func TestCreateThumbnailsFromOriginals(t *testing.T) { conf.InitializeTestData(t) - importer := NewImporter(conf.OriginalsPath) + indexer := NewIndexer(conf.OriginalsPath, conf.GetDb()) + + importer := NewImporter(conf.OriginalsPath, indexer) importer.ImportPhotosFromDirectory(conf.ImportPath)