beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Database: Add manual schema migrations #319

Browse source
This commit is contained in:
Michael Mayer 2021-11-21 14:05:07 +01:00
parent 0633436235
commit cdd7df8e62
19 changed files with 470 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
internal/commands/migrate.go
View file

@ -29,13 +29,13 @@ func migrateAction(ctx *cli.Context) error {
return err
}
log.Infoln("migrating database")
log.Infoln("migrating database schema...")
conf.InitDb()
elapsed := time.Since(start)
log.Infof("database migration completed in %s", elapsed)
log.Infof("migration completed in %s", elapsed)
conf.Shutdown()

23
internal/entity/cell.go
View file

@ -98,13 +98,14 @@ func (m *Cell) Refresh(api string) (err error) {
PhotoCount: 1,
}
// Create or update place.
if err = place.Save(); err != nil {
log.Warnf("place: failed updating %s [%s]", place.ID, time.Since(start))
} else {
m.Place = &place
m.PlaceID = l.PlaceID()
log.Tracef("place: updated %s [%s]", place.ID, time.Since(start))
// Create or update place.
if err = place.Save(); err != nil {
log.Errorf("index: %s while saving place %s", err, place.ID)
} else {
log.Tracef("index: updated place %s", place.ID)
}
m.CellName = l.Name()
@ -116,16 +117,18 @@ func (m *Cell) Refresh(api string) (err error) {
err = m.Save()
if err != nil {
log.Warnf("place: failed updating %s [%s]", m.ID, time.Since(start))
log.Errorf("index: %s while updating cell %s [%s]", err, m.ID, time.Since(start))
return err
} else if oldPlaceID != m.PlaceID {
err = UnscopedDb().Table(Photo{}.TableName()).
} else if oldPlaceID == m.PlaceID {
log.Tracef("index: cell %s keeps place_id %s", m.ID, m.PlaceID)
} else if err := UnscopedDb().Table(Photo{}.TableName()).
Where("place_id = ?", oldPlaceID).
UpdateColumn("place_id", m.PlaceID).
Error
Error; err != nil {
log.Warnf("index: %s while changing place_id from %s to %s", err, oldPlaceID, m.PlaceID)
}
log.Debugf("place: updated %s [%s]", m.ID, time.Since(start))
log.Debugf("index: updated cell %s [%s]", m.ID, time.Since(start))
return err
}

23
internal/entity/db_fixtures.go Normal file
View file

@ -0,0 +1,23 @@
package entity
// CreateDefaultFixtures inserts default fixtures for test and production.
func CreateDefaultFixtures() {
CreateUnknownAddress()
CreateDefaultUsers()
CreateUnknownPlace()
CreateUnknownLocation()
CreateUnknownCountry()
CreateUnknownCamera()
CreateUnknownLens()
}
// ResetTestFixtures re-creates registered database tables and inserts test fixtures.
func ResetTestFixtures() {
Entities.Migrate()
Entities.WaitForMigration()
Entities.Truncate()
CreateDefaultFixtures()
CreateTestFixtures()
}

37
internal/entity/db_migrate.go Normal file
View file

@ -0,0 +1,37 @@
package entity
// MigrateDb creates database tables and inserts default fixtures as needed.
func MigrateDb(dropDeprecated bool) {
if dropDeprecated {
DeprecatedTables.Drop()
}
Entities.Migrate()
Entities.WaitForMigration()
CreateDefaultFixtures()
}
// InitTestDb connects to and completely initializes the test database incl fixtures.
func InitTestDb(driver, dsn string) *Gorm {
if HasDbProvider() {
return nil
}
if driver == "test" || driver == "sqlite" || driver == "" || dsn == "" {
driver = "sqlite3"
dsn = ".test.db"
}
log.Infof("initializing %s test db in %s", driver, dsn)
db := &Gorm{
Driver: driver,
Dsn: dsn,
}
SetDbProvider(db)
ResetTestFixtures()
return db
}

111
internal/entity/db_tables.go Normal file
View file

@ -0,0 +1,111 @@
package entity
import (
"fmt"
"time"
"github.com/photoprism/photoprism/internal/migrate"
"github.com/photoprism/photoprism/pkg/txt"
)
type Tables map[string]interface{}
// Entities contains database entities and their table names.
var Entities = Tables{
migrate.Migration{}.TableName(): &migrate.Migration{},
"errors": &Error{},
"addresses": &Address{},
"users": &User{},
"accounts": &Account{},
"folders": &Folder{},
"duplicates": &Duplicate{},
File{}.TableName(): &File{},
"files_share": &FileShare{},
"files_sync": &FileSync{},
Photo{}.TableName(): &Photo{},
"details": &Details{},
Place{}.TableName(): &Place{},
Cell{}.TableName(): &Cell{},
"cameras": &Camera{},
"lenses": &Lens{},
"countries": &Country{},
"albums": &Album{},
"photos_albums": &PhotoAlbum{},
"labels": &Label{},
"categories": &Category{},
"photos_labels": &PhotoLabel{},
"keywords": &Keyword{},
"photos_keywords": &PhotoKeyword{},
"passwords": &Password{},
"links": &Link{},
Subject{}.TableName(): &Subject{},
Face{}.TableName(): &Face{},
Marker{}.TableName(): &Marker{},
}
// WaitForMigration waits for the database migration to be successful.
func (list Tables) WaitForMigration() {
type RowCount struct {
Count int
}
attempts := 100
for name := range list {
for i := 0; i <= attempts; i++ {
count := RowCount{}
if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
log.Tracef("entity: %s migrated", txt.Quote(name))
break
} else {
log.Debugf("entity: waiting for %s migration (%s)", txt.Quote(name), err.Error())
}
if i == attempts {
panic("migration failed")
}
time.Sleep(50 * time.Millisecond)
}
}
}
// Truncate removes all data from tables without dropping them.
func (list Tables) Truncate() {
for name := range list {
if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
// log.Debugf("entity: removed all data from %s", name)
break
} else if err.Error() != "record not found" {
log.Debugf("entity: %s in %s", err, txt.Quote(name))
}
}
}
// Migrate migrates all database tables of registered entities.
func (list Tables) Migrate() {
for name, entity := range list {
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
log.Debugf("entity: %s (waiting 1s)", err.Error())
time.Sleep(time.Second)
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
log.Errorf("entity: failed migrating %s", txt.Quote(name))
panic(err)
}
}
}
if err := migrate.Auto(Db()); err != nil {
log.Error(err)
}
}
// Drop drops all database tables of registered entities.
func (list Tables) Drop() {
for _, entity := range list {
if err := UnscopedDb().DropTableIfExists(entity).Error; err != nil {
panic(err)
}
}
}

178
internal/entity/entity.go
View file

@ -10,11 +10,6 @@ https://github.com/photoprism/photoprism/wiki/Storage
package entity
import (
"fmt"
"time"
"github.com/photoprism/photoprism/pkg/txt"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/event"
)
@ -28,176 +23,3 @@ func logError(result *gorm.DB) {
log.Error(result.Error.Error())
}
}
// TypeString returns an entity type string for logging.
func TypeString(entityType string) string {
if entityType == "" {
return "unknown"
}
return entityType
}
type Types map[string]interface{}
// Entities contains database entities and their table names.
var Entities = Types{
"errors": &Error{},
"addresses": &Address{},
"users": &User{},
"accounts": &Account{},
"folders": &Folder{},
"duplicates": &Duplicate{},
"files": &File{},
"files_share": &FileShare{},
"files_sync": &FileSync{},
"photos": &Photo{},
"details": &Details{},
"places": &Place{},
"cells": &Cell{},
"cameras": &Camera{},
"lenses": &Lens{},
"countries": &Country{},
"albums": &Album{},
"photos_albums": &PhotoAlbum{},
"labels": &Label{},
"categories": &Category{},
"photos_labels": &PhotoLabel{},
"keywords": &Keyword{},
"photos_keywords": &PhotoKeyword{},
"passwords": &Password{},
"links": &Link{},
Subject{}.TableName(): &Subject{},
Face{}.TableName(): &Face{},
Marker{}.TableName(): &Marker{},
}
type RowCount struct {
Count int
}
// WaitForMigration waits for the database migration to be successful.
func (list Types) WaitForMigration() {
attempts := 100
for name := range list {
for i := 0; i <= attempts; i++ {
count := RowCount{}
if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
log.Tracef("entity: %s migrated", txt.Quote(name))
break
} else {
log.Debugf("entity: waiting for %s migration (%s)", txt.Quote(name), err.Error())
}
if i == attempts {
panic("migration failed")
}
time.Sleep(50 * time.Millisecond)
}
}
}
// Truncate removes all data from tables without dropping them.
func (list Types) Truncate() {
for name := range list {
if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
// log.Debugf("entity: removed all data from %s", name)
break
} else if err.Error() != "record not found" {
log.Debugf("entity: %s in %s", err, txt.Quote(name))
}
}
}
// Migrate migrates all database tables of registered entities.
func (list Types) Migrate() {
for name, entity := range list {
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
log.Debugf("entity: %s (waiting 1s)", err.Error())
time.Sleep(time.Second)
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
log.Errorf("entity: failed migrating %s", txt.Quote(name))
panic(err)
}
}
}
}
// Drop drops all database tables of registered entities.
func (list Types) Drop() {
for _, entity := range list {
if err := UnscopedDb().DropTableIfExists(entity).Error; err != nil {
panic(err)
}
}
}
// CreateDefaultFixtures inserts default fixtures for test and production.
func CreateDefaultFixtures() {
CreateUnknownAddress()
CreateDefaultUsers()
CreateUnknownPlace()
CreateUnknownLocation()
CreateUnknownCountry()
CreateUnknownCamera()
CreateUnknownLens()
}
// MigrateIndexes runs additional table index migration queries.
func MigrateIndexes() {
if err := Db().Exec("DROP INDEX IF EXISTS idx_places_place_label ON places").Error; err != nil {
log.Errorf("%s: %s (drop index)", DbDialect(), err)
}
}
// MigrateDb creates database tables and inserts default fixtures as needed.
func MigrateDb(dropDeprecated bool) {
if dropDeprecated {
DeprecatedTables.Drop()
}
Entities.Migrate()
Entities.WaitForMigration()
MigrateIndexes()
CreateDefaultFixtures()
}
// ResetTestFixtures re-creates registered database tables and inserts test fixtures.
func ResetTestFixtures() {
Entities.Migrate()
Entities.WaitForMigration()
Entities.Truncate()
CreateDefaultFixtures()
CreateTestFixtures()
}
// InitTestDb connects to and completely initializes the test database incl fixtures.
func InitTestDb(driver, dsn string) *Gorm {
if HasDbProvider() {
return nil
}
if driver == "test" || driver == "sqlite" || driver == "" || dsn == "" {
driver = "sqlite3"
dsn = ".test.db"
}
log.Infof("initializing %s test db in %s", driver, dsn)
db := &Gorm{
Driver: driver,
Dsn: dsn,
}
SetDbProvider(db)
ResetTestFixtures()
return db
}

2
internal/entity/file.go
View file

@ -252,7 +252,7 @@ func (m *File) ReplaceHash(newHash string) error {
return nil
}
entities := Types{
entities := Tables{
"albums": Album{},
"labels": Label{},
}

9
internal/entity/values.go
View file

@ -71,3 +71,12 @@ func Trim(s string, maxLen int) string {
func SanitizeTypeString(s string) string {
return Trim(ToASCII(strings.ToLower(s)), TrimTypeString)
}
// TypeString returns an entity type string for logging.
func TypeString(entityType string) string {
if entityType == "" {
return "unknown"
}
return entityType
}

31
internal/migrate/auto.go Normal file
View file

@ -0,0 +1,31 @@
package migrate
import (
"fmt"
"github.com/jinzhu/gorm"
)
// Auto automatically migrates the database provided.
func Auto(db *gorm.DB) error {
if db == nil {
return fmt.Errorf("migrate: database connection required")
}
name := db.Dialect().GetName()
if name == "" {
return fmt.Errorf("migrate: database has no dialect name")
}
if err := db.AutoMigrate(&Migration{}).Error; err != nil {
return fmt.Errorf("migrate: %s (create migrations table)", err)
}
if migrations, ok := Dialects[name]; ok && len(migrations) > 0 {
migrations.Start(db)
return nil
} else {
return fmt.Errorf("migrate: no migrations found for %s", name)
}
}

10
internal/migrate/dialect_mysql.go Normal file
View file

@ -0,0 +1,10 @@
// Code generated by go generate; DO NOT EDIT.
package migrate
var DialectMySQL = Migrations{
{
ID: "20211121-094727",
Dialect: "mysql",
Query: "DROP INDEX IF EXISTS uix_places_place_label ON places;",
},
}

10
internal/migrate/dialect_sqlite.go Normal file
View file

@ -0,0 +1,10 @@
// Code generated by go generate; DO NOT EDIT.
package migrate
var DialectSQLite = Migrations{
{
ID: "20211121-094727",
Dialect: "sqlite",
Query: "DROP INDEX IF EXISTS idx_places_place_label ON places;",
},
}

12
internal/migrate/dialects.go Normal file
View file

@ -0,0 +1,12 @@
package migrate
// Supported database dialects.
const (
MySQL = "mysql"
SQLite = "sqlite3"
)
var Dialects = map[string]Migrations{
MySQL: DialectMySQL,
SQLite: DialectSQLite,
}

93
internal/migrate/generate.go Normal file
View file

@ -0,0 +1,93 @@
//go:build ignore
// +build ignore
// This generates countries.go by running "go generate"
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"text/template"
)
func gen_migrations(name string) {
if name == "" {
return
}
dialect := strings.ToLower(name)
type Migration struct {
ID string
Dialect string
Query string
}
var migrations []Migration
// Folder in which migration files are stored.
folder := "./" + dialect
// Returns directory entries sorted by filename.
files, _ := os.ReadDir(folder)
fmt.Printf("generating %s...", dialect)
// Read migrations from files.
for _, file := range files {
filePath := filepath.Join(folder, file.Name())
if file.IsDir() {
continue
} else if id := strings.SplitN(filepath.Base(file.Name()), ".", 2)[0]; id == "" {
fmt.Printf("e")
// Ignore.
} else if query, err := os.ReadFile(filePath); err == nil && len(query) > 0 {
fmt.Printf(".")
migrations = append(migrations, Migration{ID: id, Dialect: dialect, Query: string(query)})
} else {
fmt.Printf("f")
fmt.Println(err.Error())
}
}
fmt.Printf(" found %d migrations\n", len(migrations))
// Create source file from migrations.
f, err := os.Create(fmt.Sprintf("dialect_%s.go", dialect))
if err != nil {
panic(err)
}
defer f.Close()
// Render source template.
migrationsTemplate.Execute(f, struct {
Name string
Migrations []Migration
}{
Name: name,
Migrations: migrations,
})
}
func main() {
gen_migrations("MySQL")
gen_migrations("SQLite")
}
var migrationsTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
package migrate
var Dialect{{ print .Name }} = Migrations{
{{- range .Migrations }}
{
ID: {{ printf "%q" .ID }},
Dialect: {{ printf "%q" .Dialect }},
Query: {{ printf "%q" .Query }},
},
{{- end }}
}`))

44
internal/migrate/migrate.go Normal file
View file

@ -0,0 +1,44 @@
/*
Package migrate provides database schema migrations.
Copyright (c) 2018 - 2021 Michael Mayer <hello@photoprism.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
PhotoPrism® is a registered trademark of Michael Mayer. You may use it as required
to describe our software, run your own server, for educational purposes, but not for
offering commercial goods, products, or services without prior written permission.
In other words, please ask.
Feel free to send an e-mail to hello@photoprism.org if you have questions,
want to support our work, or just want to say hello.
Additional information can be found in our Developer Guide:
https://docs.photoprism.org/developer-guide/
*/
package migrate
//go:generate go run generate.go
//go:generate go fmt .
import (
"github.com/photoprism/photoprism/internal/event"
)
var log = event.Log
// Values is a shortcut for map[string]interface{}
type Values map[string]interface{}

57
internal/migrate/migration.go Normal file
View file

@ -0,0 +1,57 @@
package migrate
import (
"time"
"github.com/jinzhu/gorm"
)
// Migration represents a database schema migration.
type Migration struct {
ID string `gorm:"size:16;primary_key;auto_increment:false;" json:"ID" yaml:"ID"`
Dialect string `gorm:"size:16;" json:"Dialect" yaml:"Dialect,omitempty"`
Error string `gorm:"size:255;" json:"Error" yaml:"Error,omitempty"`
Source string `gorm:"size:16;" json:"Source" yaml:"Source,omitempty"`
Query string `gorm:"-" json:"Query" yaml:"Query,omitempty"`
StartedAt time.Time `json:"StartedAt" yaml:"StartedAt,omitempty"`
FinishedAt *time.Time `json:"FinishedAt" yaml:"FinishedAt,omitempty"`
}
// TableName returns the entity database table name.
func (Migration) TableName() string {
return "migrations"
}
// Fail marks the migration as failed by adding an error message.
func (m *Migration) Fail(err error, db *gorm.DB) {
if err == nil {
return
}
m.Error = err.Error()
db.Model(m).Updates(Values{"Error": m.Error})
}
// Finish updates the FinishedAt timestamp when the migration was successful.
func (m *Migration) Finish(db *gorm.DB) {
db.Model(m).Updates(Values{"FinishedAt": time.Now().UTC()})
}
// Execute runs the migration.
func (m *Migration) Execute(db *gorm.DB) {
start := time.Now()
m.StartedAt = start.UTC().Round(time.Second)
if err := db.Create(m).Error; err != nil {
return
}
if err := db.Exec(m.Query).Error; err != nil {
m.Fail(err, db)
log.Errorf("migration %s failed: %s [%s]", m.ID, err, time.Since(start))
} else {
m.Finish(db)
log.Infof("migration %s successful [%s]", m.ID, time.Since(start))
}
}

13
internal/migrate/migrations.go Normal file
View file

@ -0,0 +1,13 @@
package migrate
import "github.com/jinzhu/gorm"
// Migrations represents a sorted list of migrations.
type Migrations []Migration
// Start runs all migrations that haven't been executed yet.
func (m *Migrations) Start(db *gorm.DB) {
for _, migration := range *m {
migration.Execute(db)
}
}

1
internal/migrate/mysql/20211121-094727.sql Normal file
View file

@ -0,0 +1 @@
DROP INDEX IF EXISTS uix_places_place_label ON places;

1
internal/migrate/sqlite/20211121-094727.sql Normal file
View file

@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_places_place_label ON places;

2
internal/query/query.go
View file

@ -1,6 +1,6 @@
/*
Package query contains frequently used database queries for use in commands and API.
Package query provides frequently used database queries for use in commands and API.
Copyright (c) 2018 - 2021 Michael Mayer <hello@photoprism.org>