Implemented index command
This commit is contained in:
parent
595c32b856
commit
546a65aff0
|
@ -82,7 +82,7 @@ RUN mkdir -m 777 /go/pkg/dep
|
|||
# USER photoprism
|
||||
|
||||
# Set up project directory
|
||||
WORKDIR "/go/src/photoprism"
|
||||
WORKDIR "/go/src/github.com/photoprism/photoprism"
|
||||
COPY . .
|
||||
|
||||
RUN dep ensure
|
||||
|
|
32
Gopkg.lock
generated
32
Gopkg.lock
generated
|
@ -4,14 +4,14 @@
|
|||
[[projects]]
|
||||
name = "cloud.google.com/go"
|
||||
packages = ["civil"]
|
||||
revision = "777200caa7fb8936aed0f12b1fd79af64cc83ec9"
|
||||
version = "v0.24.0"
|
||||
revision = "aad3f485ee528456e0768f20397b4d9dd941e755"
|
||||
version = "v0.25.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/araddon/dateparse"
|
||||
packages = ["."]
|
||||
revision = "6135c1994ea28aa1bb341f5c704885906f584904"
|
||||
revision = "089f77b1d92b615cc77fde0d2fa528b5e85e832d"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
|
@ -38,13 +38,13 @@
|
|||
".",
|
||||
"internal/cp"
|
||||
]
|
||||
revision = "94c9c97e8c9f9844d15c846854a7a6031ae2132c"
|
||||
revision = "242fa5aa1b45aeb9fcdfeee88822982e3f548e22"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/disintegration/imaging"
|
||||
packages = ["."]
|
||||
revision = "1884593a19ddc6f2ea050403430d02c1d0fc1283"
|
||||
version = "v1.3.0"
|
||||
revision = "bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134"
|
||||
version = "v1.4.2"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/djherbis/times"
|
||||
|
@ -98,12 +98,6 @@
|
|||
revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4"
|
||||
version = "v1.9.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/photoprism/photoprism"
|
||||
packages = ["."]
|
||||
revision = "b2659ba5ce48b223490b8f51db065d93ae8f0cf5"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/pkg/errors"
|
||||
packages = ["."]
|
||||
|
@ -124,7 +118,7 @@
|
|||
"mknote",
|
||||
"tiff"
|
||||
]
|
||||
revision = "17202558c8d9c3fd047859f1a5e73fd9ae709187"
|
||||
revision = "8d986c03457a2057c7b0fb0a48113f7dd48f9619"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/steakknife/hamming"
|
||||
|
@ -135,8 +129,8 @@
|
|||
[[projects]]
|
||||
name = "github.com/stretchr/testify"
|
||||
packages = ["assert"]
|
||||
revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
|
||||
version = "v1.2.1"
|
||||
revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
|
||||
version = "v1.2.2"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/tensorflow/tensorflow"
|
||||
|
@ -144,8 +138,8 @@
|
|||
"tensorflow/go",
|
||||
"tensorflow/go/op"
|
||||
]
|
||||
revision = "37aa430d84ced579342a4044c89c236664be7f68"
|
||||
version = "v1.5.0"
|
||||
revision = "25c197e02393bd44f50079945409009dd4d434f8"
|
||||
version = "v1.9.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/urfave/cli"
|
||||
|
@ -171,7 +165,7 @@
|
|||
"vp8l",
|
||||
"webp"
|
||||
]
|
||||
revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
|
||||
revision = "c73c2afc3b812cdd6385de5a50616511c4a3d458"
|
||||
|
||||
[[projects]]
|
||||
name = "google.golang.org/appengine"
|
||||
|
@ -182,6 +176,6 @@
|
|||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "8aa59b793f2c56ca48723acc6a2b517cbd2dd323af673b71c1bcc74d491951bf"
|
||||
inputs-digest = "a6f9a83ff2c8ea983a59f7664e5594f0b44e75bf123a6e5b4643574e5ec1765f"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
|
|
@ -14,7 +14,7 @@ func main() {
|
|||
app := cli.NewApp()
|
||||
app.Name = "PhotoPrism"
|
||||
app.Usage = "Digital Photo Archive"
|
||||
app.Version = "0.1.0"
|
||||
app.Version = "0.2.0"
|
||||
app.Flags = globalCliFlags
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
|
@ -65,7 +65,9 @@ func main() {
|
|||
|
||||
fmt.Printf("Importing photos from %s...\n", conf.ImportPath)
|
||||
|
||||
importer := photoprism.NewImporter(conf.OriginalsPath, conf.GetDb())
|
||||
indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
importer := photoprism.NewImporter(conf.OriginalsPath, indexer)
|
||||
|
||||
importer.ImportPhotosFromDirectory(conf.ImportPath)
|
||||
|
||||
|
@ -74,6 +76,27 @@ func main() {
|
|||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "index",
|
||||
Usage: "Re-indexes all originals",
|
||||
Action: func(context *cli.Context) error {
|
||||
conf.SetValuesFromFile(photoprism.GetExpandedFilename(context.GlobalString("config-file")))
|
||||
|
||||
conf.SetValuesFromCliContext(context)
|
||||
|
||||
conf.CreateDirectories()
|
||||
|
||||
fmt.Printf("Indexing photos in %s...\n", conf.OriginalsPath)
|
||||
|
||||
indexer := photoprism.NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
indexer.IndexAll()
|
||||
|
||||
fmt.Println("Done.")
|
||||
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "convert",
|
||||
Usage: "Converts RAW originals to JPEG",
|
||||
|
|
|
@ -4,4 +4,4 @@ thumbnails-path: photos/thumbnails
|
|||
import-path: photos/import
|
||||
export-path: photos/export
|
||||
database-driver: mysql
|
||||
database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism
|
||||
database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true
|
|
@ -19,7 +19,7 @@ var thumbnailsPath = GetExpandedFilename(testDataPath + "/thumbnails")
|
|||
var importPath = GetExpandedFilename(testDataPath + "/import")
|
||||
var exportPath = GetExpandedFilename(testDataPath + "/export")
|
||||
var databaseDriver = "mysql"
|
||||
var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism"
|
||||
var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism?parseTime=true"
|
||||
|
||||
func (c *Config) RemoveTestData(t *testing.T) {
|
||||
os.RemoveAll(c.ImportPath)
|
||||
|
|
|
@ -7,7 +7,7 @@ services:
|
|||
- 80:80
|
||||
- 8080:8080
|
||||
volumes:
|
||||
- .:/go/src/photoprism
|
||||
- .:/go/src/github.com/photoprism/photoprism
|
||||
|
||||
database:
|
||||
image: mysql:latest
|
||||
|
|
12
importer.go
12
importer.go
|
@ -2,7 +2,6 @@ package photoprism
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/pkg/errors"
|
||||
"log"
|
||||
"os"
|
||||
|
@ -14,16 +13,16 @@ import (
|
|||
|
||||
type Importer struct {
|
||||
originalsPath string
|
||||
db *gorm.DB
|
||||
indexer *Indexer
|
||||
removeDotFiles bool
|
||||
removeExistingFiles bool
|
||||
removeEmptyDirectories bool
|
||||
}
|
||||
|
||||
func NewImporter(originalsPath string, db *gorm.DB) *Importer {
|
||||
func NewImporter(originalsPath string, indexer *Indexer) *Importer {
|
||||
instance := &Importer{
|
||||
originalsPath: originalsPath,
|
||||
db: db,
|
||||
indexer: indexer,
|
||||
removeDotFiles: true,
|
||||
removeExistingFiles: true,
|
||||
removeEmptyDirectories: true,
|
||||
|
@ -66,14 +65,13 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
|
|||
os.MkdirAll(path.Dir(destinationFilename), os.ModePerm)
|
||||
log.Printf("Moving file %s to %s", relatedMediaFile.GetFilename(), destinationFilename)
|
||||
relatedMediaFile.Move(destinationFilename)
|
||||
i.indexer.IndexMediaFile(relatedMediaFile)
|
||||
} else if i.removeExistingFiles {
|
||||
relatedMediaFile.Remove()
|
||||
log.Printf("Deleted %s (already exists)", relatedMediaFile.GetFilename())
|
||||
}
|
||||
}
|
||||
|
||||
// mediaFile.Move(i.originalsPath)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
|
@ -115,7 +113,7 @@ func (i *Importer) GetDestinationFilename(masterFile *MediaFile, mediaFile *Medi
|
|||
|
||||
iteration++
|
||||
|
||||
result = pathName + "/" + canonicalName + "_" + fmt.Sprintf("V%d", iteration) + fileExtension
|
||||
result = pathName + "/" + canonicalName + "." + fmt.Sprintf("V%d", iteration) + fileExtension
|
||||
}
|
||||
|
||||
return result, nil
|
||||
|
|
|
@ -8,7 +8,9 @@ import (
|
|||
func TestNewImporter(t *testing.T) {
|
||||
conf := NewTestConfig()
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath)
|
||||
indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath, indexer)
|
||||
|
||||
assert.IsType(t, &Importer{}, importer)
|
||||
}
|
||||
|
@ -18,7 +20,9 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) {
|
|||
|
||||
conf.InitializeTestData(t)
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath)
|
||||
indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath, indexer)
|
||||
|
||||
importer.ImportPhotosFromDirectory(conf.ImportPath)
|
||||
}
|
||||
|
@ -26,7 +30,10 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) {
|
|||
func TestImporter_GetDestinationFilename(t *testing.T) {
|
||||
conf := NewTestConfig()
|
||||
conf.InitializeTestData(t)
|
||||
importer := NewImporter(conf.OriginalsPath)
|
||||
|
||||
indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath, indexer)
|
||||
|
||||
rawFile := NewMediaFile(conf.ImportPath + "/raw/IMG_1435.CR2")
|
||||
|
||||
|
@ -34,5 +41,5 @@ func TestImporter_GetDestinationFilename(t *testing.T) {
|
|||
|
||||
assert.Empty(t, err)
|
||||
|
||||
assert.Equal(t, conf.OriginalsPath+"/2018/02/20180204_180813_B0770443A5F7.cr2", filename)
|
||||
assert.Equal(t, conf.OriginalsPath + "/2018/02/20180204_170813_B0770443A5F7.cr2", filename)
|
||||
}
|
||||
|
|
118
indexer.go
118
indexer.go
|
@ -1 +1,119 @@
|
|||
package photoprism
|
||||
|
||||
import (
|
||||
"github.com/jinzhu/gorm"
|
||||
"os"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"github.com/photoprism/photoprism/recognize"
|
||||
)
|
||||
|
||||
type Indexer struct {
|
||||
originalsPath string
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewIndexer(originalsPath string, db *gorm.DB) *Indexer {
|
||||
instance := &Indexer{
|
||||
originalsPath: originalsPath,
|
||||
db: db,
|
||||
}
|
||||
|
||||
return instance
|
||||
}
|
||||
|
||||
func (i *Indexer) GetImageTags(jpeg *MediaFile) (result []Tag) {
|
||||
if imageBuffer, err := ioutil.ReadFile(jpeg.filename); err == nil {
|
||||
tags, err := recognize.GetImageTags(string(imageBuffer))
|
||||
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
for _, tag := range tags {
|
||||
if tag.Probability > 0.2 {
|
||||
result = append(result, Tag{Label: tag.Label})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (i *Indexer) IndexMediaFile(mediaFile *MediaFile) {
|
||||
var photo Photo
|
||||
var file File
|
||||
|
||||
canonicalName := mediaFile.GetCanonicalNameFromFile()
|
||||
fileHash := mediaFile.GetHash()
|
||||
|
||||
if result := i.db.First(&photo, "canonical_name = ?", canonicalName); result.Error != nil {
|
||||
if jpeg, err := mediaFile.GetJpeg(); err == nil {
|
||||
if perceptualHash, err := jpeg.GetPerceptualHash(); err == nil {
|
||||
photo.PerceptualHash = perceptualHash
|
||||
}
|
||||
|
||||
if exifData, err := jpeg.GetExifData(); err == nil {
|
||||
photo.Lat = exifData.Lat
|
||||
photo.Long = exifData.Long
|
||||
}
|
||||
|
||||
photo.Tags = i.GetImageTags(jpeg)
|
||||
}
|
||||
|
||||
photo.CanonicalName = canonicalName
|
||||
photo.Files = []File{}
|
||||
photo.Albums = []Album{}
|
||||
photo.Author = ""
|
||||
photo.CameraModel = mediaFile.GetCameraModel()
|
||||
photo.LocationName = ""
|
||||
photo.Liked = false
|
||||
photo.Private = true
|
||||
photo.Deleted = false
|
||||
|
||||
i.db.Create(&photo)
|
||||
}
|
||||
|
||||
if result := i.db.First(&file, "hash = ?", fileHash); result.Error != nil {
|
||||
file.PhotoID = photo.ID
|
||||
file.Filename = mediaFile.GetFilename()
|
||||
file.Hash = fileHash
|
||||
file.FileType = mediaFile.GetType()
|
||||
file.MimeType = mediaFile.GetMimeType()
|
||||
|
||||
i.db.Create(&file)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Indexer) IndexAll() {
|
||||
err := filepath.Walk(i.originalsPath, func(filename string, fileInfo os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if fileInfo.IsDir() || strings.HasPrefix(filepath.Base(filename), ".") {
|
||||
return nil
|
||||
}
|
||||
|
||||
mediaFile := NewMediaFile(filename)
|
||||
|
||||
if !mediaFile.Exists() || !mediaFile.IsPhoto() {
|
||||
return nil
|
||||
}
|
||||
|
||||
relatedFiles, _, _ := mediaFile.GetRelatedFiles()
|
||||
|
||||
for _, relatedMediaFile := range relatedFiles {
|
||||
log.Printf("Indexing %s", relatedMediaFile.GetFilename())
|
||||
i.IndexMediaFile(relatedMediaFile)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
}
|
||||
|
|
34
mediafile.go
34
mediafile.go
|
@ -12,6 +12,7 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -38,6 +39,7 @@ var FileExtensions = map[string]string{
|
|||
".avi": FileTypeMovie,
|
||||
".yml": FileTypeYaml,
|
||||
".jpg": FileTypeJpeg,
|
||||
".thm": FileTypeJpeg,
|
||||
".jpeg": FileTypeJpeg,
|
||||
".xmp": FileTypeXmp,
|
||||
".aae": FileTypeAae,
|
||||
|
@ -110,6 +112,16 @@ func (m *MediaFile) GetCanonicalName() string {
|
|||
return result
|
||||
}
|
||||
|
||||
func (m *MediaFile) GetCanonicalNameFromFile() string {
|
||||
basename := filepath.Base(m.GetFilename())
|
||||
|
||||
if end := strings.Index(basename, "."); end != -1 {
|
||||
return basename[:end] // Length of canonical name: 16 + 12
|
||||
} else {
|
||||
return basename
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MediaFile) GetPerceptualHash() (string, error) {
|
||||
if m.perceptualHash != "" {
|
||||
return m.perceptualHash, nil
|
||||
|
@ -297,12 +309,16 @@ func (m *MediaFile) IsJpeg() bool {
|
|||
return m.GetMimeType() == MimeTypeJpeg
|
||||
}
|
||||
|
||||
func (m *MediaFile) GetType() string {
|
||||
return FileExtensions[m.GetExtension()]
|
||||
}
|
||||
|
||||
func (m *MediaFile) HasType(typeString string) bool {
|
||||
if typeString == FileTypeJpeg {
|
||||
return m.IsJpeg()
|
||||
}
|
||||
|
||||
return FileExtensions[m.GetExtension()] == typeString
|
||||
return m.GetType() == typeString
|
||||
}
|
||||
|
||||
func (m *MediaFile) IsRaw() bool {
|
||||
|
@ -312,3 +328,19 @@ func (m *MediaFile) IsRaw() bool {
|
|||
func (m *MediaFile) IsPhoto() bool {
|
||||
return m.IsJpeg() || m.IsRaw()
|
||||
}
|
||||
|
||||
func (m *MediaFile) GetJpeg() (*MediaFile, error) {
|
||||
if m.IsJpeg() {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
jpegFilename := m.GetFilename()[0:len(m.GetFilename()) - len(filepath.Ext(m.GetFilename()))] + ".jpg"
|
||||
|
||||
if !fileExists(jpegFilename) {
|
||||
return nil, errors.New("file does not exist")
|
||||
}
|
||||
|
||||
result := NewMediaFile(jpegFilename)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
|
Before Width: | Height: | Size: 130 KiB After Width: | Height: | Size: 130 KiB |
|
@ -1,4 +1,4 @@
|
|||
package tensorflow
|
||||
package recognize
|
||||
|
||||
import (
|
||||
tf "github.com/tensorflow/tensorflow/tensorflow/go"
|
|
@ -1,4 +1,4 @@
|
|||
package tensorflow
|
||||
package recognize
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
|
@ -25,7 +25,7 @@ var (
|
|||
labels []string
|
||||
)
|
||||
|
||||
func RecognizeImage(image string) (result []LabelResult, err error) {
|
||||
func GetImageTags(image string) (result []LabelResult, err error) {
|
||||
if err := loadModel(); err != nil {
|
||||
return nil, err
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package tensorflow
|
||||
package recognize
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
@ -6,11 +6,11 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRecognizeImage(t *testing.T) {
|
||||
func TestGetImageTags(t *testing.T) {
|
||||
if imageBuffer, err := ioutil.ReadFile("cat.jpg"); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
result, err := RecognizeImage(string(imageBuffer))
|
||||
result, err := GetImageTags(string(imageBuffer))
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Nil(t, err)
|
|
@ -44,7 +44,9 @@ func TestCreateThumbnailsFromOriginals(t *testing.T) {
|
|||
|
||||
conf.InitializeTestData(t)
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath)
|
||||
indexer := NewIndexer(conf.OriginalsPath, conf.GetDb())
|
||||
|
||||
importer := NewImporter(conf.OriginalsPath, indexer)
|
||||
|
||||
importer.ImportPhotosFromDirectory(conf.ImportPath)
|
||||
|
||||
|
|
Loading…
Reference in a new issue