diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index b5e21f356..840741b42 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -38,6 +38,7 @@ services: PHOTOPRISM_DATABASE_PASSWORD: "photoprism" PHOTOPRISM_TEST_DRIVER: "sqlite" PHOTOPRISM_TEST_DSN: ".test.db" + PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true" PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters) PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets" PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage" @@ -157,6 +158,21 @@ services: MYSQL_PASSWORD: photoprism MYSQL_DATABASE: photoprism + ## MySQL Database Server + ## Docs: https://dev.mysql.com/doc/refman/8.0/en/ + mysql: + image: mysql:8 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120 + expose: + - "4001" # Database port (internal) + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + ## Dummy WebDAV Server dummy-webdav: image: photoprism/dummy-webdav:20211109 diff --git a/docker-compose.db.yml b/docker-compose.mariadb.yml similarity index 81% rename from docker-compose.db.yml rename to docker-compose.mariadb.yml index 2ab8423f0..1ae08a53a 100644 --- a/docker-compose.db.yml +++ b/docker-compose.mariadb.yml @@ -57,19 +57,6 @@ services: MYSQL_PASSWORD: photoprism MYSQL_DATABASE: photoprism - mysql-8: - image: mysql:8 - command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120 - expose: - - "4001" # Database port (internal) - volumes: - - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" - environment: - MYSQL_ROOT_PASSWORD: photoprism - MYSQL_USER: photoprism - MYSQL_PASSWORD: photoprism - MYSQL_DATABASE: photoprism - networks: default: external: diff --git a/docker-compose.yml b/docker-compose.yml index 61af7c38e..c5e8d6ffe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: - "2343:2343" # Acceptance Test HTTP port (host:container) shm_size: "2gb" environment: + PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters) PHOTOPRISM_UID: ${UID:-1000} PHOTOPRISM_GID: ${GID:-1000} PHOTOPRISM_SITE_URL: "http://localhost:2342/" @@ -44,7 +45,7 @@ services: PHOTOPRISM_DATABASE_PASSWORD: "photoprism" PHOTOPRISM_TEST_DRIVER: "sqlite" PHOTOPRISM_TEST_DSN: ".test.db" - PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters) + PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true" PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets" PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage" PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals" @@ -103,6 +104,21 @@ services: MYSQL_PASSWORD: photoprism MYSQL_DATABASE: photoprism + ## MySQL Database Server + ## Docs: https://dev.mysql.com/doc/refman/8.0/en/ + mysql: + image: mysql:8 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120 + expose: + - "4001" # Database port (internal) + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + ## Dummy WebDAV Server dummy-webdav: image: photoprism/dummy-webdav:20211109 diff --git a/internal/commands/migrate.go b/internal/commands/migrate.go index 97d543a2c..b2b6bd511 100644 --- a/internal/commands/migrate.go +++ b/internal/commands/migrate.go @@ -4,15 +4,26 @@ import ( "context" "time" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "github.com/photoprism/photoprism/internal/config" ) -// MigrateCommand registers the migrate cli command. +// MigrateCommand registers the "migrate" CLI command. var MigrateCommand = cli.Command{ - Name: "migrate", - Usage: "Updates the index database schema", + Name: "migrate", + Usage: "Updates the index database schema", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "failed, f", + Usage: "run previously failed migrations", + }, + cli.BoolFlag{ + Name: "trace, t", + Usage: "show trace logs for debugging", + }, + }, Action: migrateAction, } @@ -29,15 +40,26 @@ func migrateAction(ctx *cli.Context) error { return err } + defer conf.Shutdown() + + if ctx.Bool("trace") { + log.SetLevel(logrus.TraceLevel) + log.Infoln("migrate: enabled trace mode") + } + + runFailed := ctx.Bool("failed") + + if runFailed { + log.Infoln("migrate: running previously failed migrations") + } + log.Infoln("migrating database schema...") - conf.InitDb() + conf.MigrateDb(runFailed) elapsed := time.Since(start) log.Infof("migration completed in %s", elapsed) - conf.Shutdown() - return nil } diff --git a/internal/commands/places.go b/internal/commands/places.go index fee027243..c691e86be 100644 --- a/internal/commands/places.go +++ b/internal/commands/places.go @@ -16,11 +16,18 @@ import ( // PlacesCommand registers the places subcommands. var PlacesCommand = cli.Command{ Name: "places", - Usage: "Geographic data subcommands", + Usage: "Maps and location details subcommands", Subcommands: []cli.Command{ { - Name: "update", - Usage: "Retrieves updated location details", + Name: "update", + Usage: "Retrieves updated location details", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "yes, y", + Hidden: true, + Usage: "assume \"yes\" as answer to all prompts and run non-interactively", + }, + }, Action: placesUpdateAction, }, }, @@ -46,14 +53,16 @@ func placesUpdateAction(ctx *cli.Context) error { return nil } - confirmPrompt := promptui.Prompt{ - Label: "Interrupting the update may result in inconsistent location details. Proceed?", - IsConfirm: true, - } + if !ctx.Bool("yes") { + confirmPrompt := promptui.Prompt{ + Label: "Interrupting the update may result in inconsistent location details. Proceed?", + IsConfirm: true, + } - // Abort? - if _, err := confirmPrompt.Run(); err != nil { - return nil + // Abort? + if _, err := confirmPrompt.Run(); err != nil { + return nil + } } start := time.Now() diff --git a/internal/commands/reset.go b/internal/commands/reset.go index f9ee50ce4..0cba82d5f 100644 --- a/internal/commands/reset.go +++ b/internal/commands/reset.go @@ -9,6 +9,7 @@ import ( "time" "github.com/manifoldco/promptui" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "github.com/photoprism/photoprism/internal/config" @@ -17,20 +18,27 @@ import ( // ResetCommand resets the index and removes sidecar files after confirmation. var ResetCommand = cli.Command{ - Name: "reset", - Usage: "Resets the index and removes generated sidecar files", + Name: "reset", + Usage: "Resets the index and removes generated sidecar files", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "index, i", + Usage: "reset index database only", + }, + cli.BoolFlag{ + Name: "trace, t", + Usage: "show trace logs for debugging", + }, + cli.BoolFlag{ + Name: "yes, y", + Usage: "assume \"yes\" as answer to all prompts and run non-interactively", + }, + }, Action: resetAction, } // resetAction resets the index and removes sidecar files after confirmation. func resetAction(ctx *cli.Context) error { - log.Warnf("YOU ARE ABOUT TO RESET THE INDEX AND REMOVE ALL JSON / YAML SIDECAR FILES") - - removeIndexPrompt := promptui.Prompt{ - Label: "Reset index database including albums and metadata?", - IsConfirm: true, - } - conf := config.NewConfig(ctx) _, cancel := context.WithCancel(context.Background()) defer cancel() @@ -39,101 +47,73 @@ func resetAction(ctx *cli.Context) error { return err } + defer conf.Shutdown() + entity.SetDbProvider(conf) - if _, err := removeIndexPrompt.Run(); err == nil { - start := time.Now() + if !ctx.Bool("yes") { + log.Warnf("This will delete and recreate your index database after confirmation") - tables := entity.Entities + if !ctx.Bool("index") { + log.Warnf("You will be asked next if you also want to remove all JSON and YAML backup files") + } + } - log.Infoln("dropping existing tables") - tables.Drop() + if ctx.Bool("trace") { + log.SetLevel(logrus.TraceLevel) + log.Infoln("reset: enabled trace mode") + } - log.Infoln("restoring default schema") - entity.MigrateDb(true) + resetIndex := ctx.Bool("yes") - if conf.AdminPassword() != "" { - log.Infoln("restoring initial admin password") - entity.Admin.InitPassword(conf.AdminPassword()) + // Show prompt? + if !resetIndex { + removeIndexPrompt := promptui.Prompt{ + Label: "Delete and recreate index database?", + IsConfirm: true, } - log.Infof("database reset completed in %s", time.Since(start)) - } else { - log.Infof("keeping index database") + if _, err := removeIndexPrompt.Run(); err == nil { + resetIndex = true + } else { + log.Infof("keeping index database") + } + } + + // Reset index? + if resetIndex { + resetIndexDb(conf) + } + + // Reset index only? + if ctx.Bool("index") { + return nil } removeSidecarJsonPrompt := promptui.Prompt{ - Label: "Permanently delete existing JSON metadata sidecar files?", + Label: "Delete all JSON metadata sidecar files?", IsConfirm: true, } if _, err := removeSidecarJsonPrompt.Run(); err == nil { - start := time.Now() - - matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json") - - if err != nil { - return err - } - - if len(matches) > 0 { - log.Infof("removing %d JSON metadata sidecar files", len(matches)) - - for _, name := range matches { - if err := os.Remove(name); err != nil { - fmt.Print("E") - } else { - fmt.Print(".") - } - } - - fmt.Println("") - - log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start)) - } else { - log.Infof("found no JSON metadata sidecar files") - } + resetSidecarJson(conf) } else { log.Infof("keeping JSON metadata sidecar files") } removeSidecarYamlPrompt := promptui.Prompt{ - Label: "Permanently delete existing YAML metadata backups?", + Label: "Delete all YAML metadata backup files?", IsConfirm: true, } if _, err := removeSidecarYamlPrompt.Run(); err == nil { - start := time.Now() - - matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml") - - if err != nil { - return err - } - - if len(matches) > 0 { - log.Infof("%d YAML metadata backups will be removed", len(matches)) - - for _, name := range matches { - if err := os.Remove(name); err != nil { - fmt.Print("E") - } else { - fmt.Print(".") - } - } - - fmt.Println("") - - log.Infof("removed all YAML metadata backups [%s]", time.Since(start)) - } else { - log.Infof("found no YAML metadata backups") - } + resetSidecarYaml(conf) } else { log.Infof("keeping YAML metadata backups") } removeAlbumYamlPrompt := promptui.Prompt{ - Label: "Permanently delete existing YAML album backups?", + Label: "Delete all YAML album backup files?", IsConfirm: true, } @@ -167,7 +147,85 @@ func resetAction(ctx *cli.Context) error { log.Infof("keeping YAML album backup files") } - conf.Shutdown() - return nil } + +// resetIndexDb resets the index database schema. +func resetIndexDb(conf *config.Config) { + start := time.Now() + + tables := entity.Entities + + log.Infoln("dropping existing tables") + tables.Drop(conf.Db()) + + log.Infoln("restoring default schema") + entity.MigrateDb(true, false) + + if conf.AdminPassword() != "" { + log.Infoln("restoring initial admin password") + entity.Admin.InitPassword(conf.AdminPassword()) + } + + log.Infof("database reset completed in %s", time.Since(start)) +} + +// resetSidecarJson removes generated JSON sidecar files. +func resetSidecarJson(conf *config.Config) { + start := time.Now() + + matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json") + + if err != nil { + log.Errorf("reset: %s (find json sidecar files)", err) + return + } + + if len(matches) > 0 { + log.Infof("removing %d JSON metadata sidecar files", len(matches)) + + for _, name := range matches { + if err := os.Remove(name); err != nil { + fmt.Print("E") + } else { + fmt.Print(".") + } + } + + fmt.Println("") + + log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start)) + } else { + log.Infof("found no JSON metadata sidecar files") + } +} + +// resetSidecarYaml removes generated YAML sidecar files. +func resetSidecarYaml(conf *config.Config) { + start := time.Now() + + matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml") + + if err != nil { + log.Errorf("reset: %s (find yaml sidecar files)", err) + return + } + + if len(matches) > 0 { + log.Infof("%d YAML metadata backups will be removed", len(matches)) + + for _, name := range matches { + if err := os.Remove(name); err != nil { + fmt.Print("E") + } else { + fmt.Print(".") + } + } + + fmt.Println("") + + log.Infof("removed all YAML metadata backups [%s]", time.Since(start)) + } else { + log.Infof("found no YAML metadata backups") + } +} diff --git a/internal/commands/restore.go b/internal/commands/restore.go index 030f3e50b..3791b3548 100644 --- a/internal/commands/restore.go +++ b/internal/commands/restore.go @@ -12,7 +12,6 @@ import ( "time" "github.com/dustin/go-humanize/english" - "github.com/urfave/cli" "github.com/photoprism/photoprism/internal/config" @@ -151,7 +150,7 @@ func restoreAction(ctx *cli.Context) error { ) case config.SQLite: log.Infoln("dropping existing tables") - tables.Drop() + tables.Drop(conf.Db()) cmd = exec.Command( conf.SqliteBin(), conf.DatabaseDsn(), diff --git a/internal/config/db.go b/internal/config/db.go index 4cf1cf634..88e718f0a 100644 --- a/internal/config/db.go +++ b/internal/config/db.go @@ -232,11 +232,16 @@ func (c *Config) SetDbOptions() { } } -// InitDb will initialize the database connection and schema. +// InitDb initializes the database without running previously failed migrations. func (c *Config) InitDb() { + c.MigrateDb(false) +} + +// MigrateDb initializes the database and migrates the schema if needed. +func (c *Config) MigrateDb(runFailed bool) { c.SetDbOptions() entity.SetDbProvider(c) - entity.MigrateDb(false) + entity.MigrateDb(true, runFailed) entity.Admin.InitPassword(c.AdminPassword()) diff --git a/internal/entity/db_fixtures.go b/internal/entity/db_fixtures.go index 9072c0335..860966d62 100644 --- a/internal/entity/db_fixtures.go +++ b/internal/entity/db_fixtures.go @@ -13,9 +13,9 @@ func CreateDefaultFixtures() { // ResetTestFixtures re-creates registered database tables and inserts test fixtures. func ResetTestFixtures() { - Entities.Migrate() - Entities.WaitForMigration() - Entities.Truncate() + Entities.Migrate(Db(), false) + Entities.WaitForMigration(Db()) + Entities.Truncate(Db()) CreateDefaultFixtures() diff --git a/internal/entity/db_migrate.go b/internal/entity/db_migrate.go index bc1d71a7a..b40b5f262 100644 --- a/internal/entity/db_migrate.go +++ b/internal/entity/db_migrate.go @@ -1,13 +1,13 @@ package entity // MigrateDb creates database tables and inserts default fixtures as needed. -func MigrateDb(dropDeprecated bool) { +func MigrateDb(dropDeprecated, runFailed bool) { if dropDeprecated { - DeprecatedTables.Drop() + DeprecatedTables.Drop(Db()) } - Entities.Migrate() - Entities.WaitForMigration() + Entities.Migrate(Db(), runFailed) + Entities.WaitForMigration(Db()) CreateDefaultFixtures() } diff --git a/internal/entity/db_mysql8_test.go b/internal/entity/db_mysql8_test.go new file mode 100644 index 000000000..49a342316 --- /dev/null +++ b/internal/entity/db_mysql8_test.go @@ -0,0 +1,51 @@ +package entity + +import ( + "os" + "testing" + "time" + + "github.com/jinzhu/gorm" +) + +func TestMySQL8(t *testing.T) { + dbDriver := MySQL + dbDsn := os.Getenv("PHOTOPRISM_TEST_DSN_MYSQL8") + + db, err := gorm.Open(dbDriver, dbDsn) + + if err != nil || db == nil { + for i := 1; i <= 5; i++ { + db, err = gorm.Open(dbDriver, dbDsn) + + if db != nil && err == nil { + break + } + + time.Sleep(5 * time.Second) + } + + if err != nil || db == nil { + t.Fatal(err) + } + } + + defer db.Close() + + db.LogMode(false) + + DeprecatedTables.Drop(db) + Entities.Drop(db) + + // First migration. + Entities.Migrate(db, false) + Entities.WaitForMigration(db) + + // Second migration. + Entities.Migrate(db, false) + Entities.WaitForMigration(db) + + // Third migration with force flag. + Entities.Migrate(db, true) + Entities.WaitForMigration(db) +} diff --git a/internal/entity/db_tables.go b/internal/entity/db_tables.go index d60b85482..fb87c31c3 100644 --- a/internal/entity/db_tables.go +++ b/internal/entity/db_tables.go @@ -4,6 +4,8 @@ import ( "fmt" "time" + "github.com/jinzhu/gorm" + "github.com/photoprism/photoprism/internal/migrate" "github.com/photoprism/photoprism/pkg/txt" ) @@ -44,7 +46,7 @@ var Entities = Tables{ } // WaitForMigration waits for the database migration to be successful. -func (list Tables) WaitForMigration() { +func (list Tables) WaitForMigration(db *gorm.DB) { type RowCount struct { Count int } @@ -53,7 +55,7 @@ func (list Tables) WaitForMigration() { for name := range list { for i := 0; i <= attempts; i++ { count := RowCount{} - if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil { + if err := db.Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil { log.Tracef("entity: %s migrated", txt.Quote(name)) break } else { @@ -70,9 +72,9 @@ func (list Tables) WaitForMigration() { } // Truncate removes all data from tables without dropping them. -func (list Tables) Truncate() { +func (list Tables) Truncate(db *gorm.DB) { for name := range list { - if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil { + if err := db.Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil { // log.Debugf("entity: removed all data from %s", name) break } else if err.Error() != "record not found" { @@ -82,29 +84,29 @@ func (list Tables) Truncate() { } // Migrate migrates all database tables of registered entities. -func (list Tables) Migrate() { +func (list Tables) Migrate(db *gorm.DB, runFailed bool) { for name, entity := range list { - if err := UnscopedDb().AutoMigrate(entity).Error; err != nil { + if err := db.AutoMigrate(entity).Error; err != nil { log.Debugf("entity: %s (waiting 1s)", err.Error()) time.Sleep(time.Second) - if err := UnscopedDb().AutoMigrate(entity).Error; err != nil { + if err := db.AutoMigrate(entity).Error; err != nil { log.Errorf("entity: failed migrating %s", txt.Quote(name)) panic(err) } } } - if err := migrate.Auto(Db()); err != nil { + if err := migrate.Auto(db, runFailed); err != nil { log.Error(err) } } // Drop drops all database tables of registered entities. -func (list Tables) Drop() { +func (list Tables) Drop(db *gorm.DB) { for _, entity := range list { - if err := UnscopedDb().DropTableIfExists(entity).Error; err != nil { + if err := db.DropTableIfExists(entity).Error; err != nil { panic(err) } } diff --git a/internal/entity/deprecated.go b/internal/entity/deprecated.go index 6c71b4330..9ff1ab67c 100644 --- a/internal/entity/deprecated.go +++ b/internal/entity/deprecated.go @@ -1,12 +1,14 @@ package entity +import "github.com/jinzhu/gorm" + // Deprecated represents a list of deprecated database tables. type Deprecated []string // Drop drops all deprecated tables. -func (list Deprecated) Drop() { +func (list Deprecated) Drop(db *gorm.DB) { for _, tableName := range list { - if err := UnscopedDb().DropTableIfExists(tableName).Error; err != nil { + if err := db.DropTableIfExists(tableName).Error; err != nil { log.Debugf("drop %s: %s", tableName, err) } } diff --git a/internal/migrate/auto.go b/internal/migrate/auto.go index 4f45e08cb..a95924925 100644 --- a/internal/migrate/auto.go +++ b/internal/migrate/auto.go @@ -7,7 +7,7 @@ import ( ) // Auto automatically migrates the database provided. -func Auto(db *gorm.DB) error { +func Auto(db *gorm.DB, runFailed bool) error { if db == nil { return fmt.Errorf("migrate: database connection required") } @@ -23,7 +23,7 @@ func Auto(db *gorm.DB) error { } if migrations, ok := Dialects[name]; ok && len(migrations) > 0 { - migrations.Start(db) + migrations.Start(db, runFailed) return nil } else { return fmt.Errorf("migrate: no migrations found for %s", name) diff --git a/internal/migrate/dialect_mysql.go b/internal/migrate/dialect_mysql.go index 19e4e767d..2b9c42b6e 100644 --- a/internal/migrate/dialect_mysql.go +++ b/internal/migrate/dialect_mysql.go @@ -5,11 +5,11 @@ var DialectMySQL = Migrations{ { ID: "20211121-094727", Dialect: "mysql", - Statements: []string{"DROP INDEX IF EXISTS uix_places_place_label ON `places`;"}, + Statements: []string{"DROP INDEX uix_places_place_label ON `places`;"}, }, { ID: "20211124-120008", Dialect: "mysql", - Statements: []string{"DROP INDEX IF EXISTS idx_places_place_label ON `places`;", "DROP INDEX IF EXISTS uix_places_label ON `places`;"}, + Statements: []string{"DROP INDEX idx_places_place_label ON `places`;", "DROP INDEX uix_places_label ON `places`;"}, }, } diff --git a/internal/migrate/migration.go b/internal/migrate/migration.go index 101ba0d7f..e67dd08d1 100644 --- a/internal/migrate/migration.go +++ b/internal/migrate/migration.go @@ -1,6 +1,7 @@ package migrate import ( + "strings" "time" "github.com/jinzhu/gorm" @@ -29,6 +30,7 @@ func (m *Migration) Fail(err error, db *gorm.DB) { } m.Error = err.Error() + db.Model(m).Updates(Values{"Error": m.Error}) } @@ -41,7 +43,11 @@ func (m *Migration) Finish(db *gorm.DB) error { func (m *Migration) Execute(db *gorm.DB) error { for _, s := range m.Statements { if err := db.Exec(s).Error; err != nil { - return err + if strings.HasPrefix(s, "DROP ") && strings.Contains(err.Error(), "DROP") { + log.Tracef("migrate: %s (drop statement)", err) + } else { + return err + } } } diff --git a/internal/migrate/migrations.go b/internal/migrate/migrations.go index 8537b32b4..c402a1298 100644 --- a/internal/migrate/migrations.go +++ b/internal/migrate/migrations.go @@ -1,33 +1,84 @@ package migrate import ( + "database/sql" "time" + "github.com/dustin/go-humanize/english" "github.com/jinzhu/gorm" ) // Migrations represents a sorted list of migrations. type Migrations []Migration +// MigrationMap represents a map of migrations. +type MigrationMap map[string]Migration + +// Existing finds and returns previously executed database schema migrations. +func Existing(db *gorm.DB) MigrationMap { + result := make(MigrationMap) + dialect := db.Dialect().GetName() + + stmt := db.Model(Migration{}).Where("dialect = ?", dialect) + stmt = stmt.Select("id, dialect, error, source, started_at, finished_at") + + rows, err := stmt.Rows() + + if err != nil { + log.Warnf("migrate: %s (find existing)", err) + return result + } + + defer func(rows *sql.Rows) { + err = rows.Close() + }(rows) + + for rows.Next() { + m := Migration{} + + if err = rows.Scan(&m.ID, &m.Dialect, &m.Error, &m.Source, &m.StartedAt, &m.FinishedAt); err != nil { + log.Warnf("migrate: %s (scan existing)", err) + return result + } + + result[m.ID] = m + } + + return result +} + // Start runs all migrations that haven't been executed yet. -func (m *Migrations) Start(db *gorm.DB) { +func (m *Migrations) Start(db *gorm.DB, runFailed bool) { + // Find previously executed migrations. + executed := Existing(db) + + log.Debugf("migrate: found %s", english.Plural(len(executed), "previous migration", "previous migrations")) + for _, migration := range *m { start := time.Now() - migration.StartedAt = start.UTC().Round(time.Second) - // Continue if already executed. - if err := db.Create(migration).Error; err != nil { + // Already executed? + if done, ok := executed[migration.ID]; ok { + // Try to run failed migrations again? + if !runFailed || done.Error == "" { + log.Debugf("migrate: %s skipped", migration.ID) + continue + } + } else if err := db.Create(migration).Error; err != nil { + // Should not happen. + log.Warnf("migrate: creating %s failed with %s [%s]", migration.ID, err, time.Since(start)) continue } + // Run migration. if err := migration.Execute(db); err != nil { migration.Fail(err, db) - log.Errorf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start)) + log.Errorf("migrate: executing %s failed with %s [%s]", migration.ID, err, time.Since(start)) } else if err = migration.Finish(db); err != nil { - log.Warnf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start)) + log.Warnf("migrate: updating %s failed with %s [%s]", migration.ID, err, time.Since(start)) } else { - log.Infof("migration %s successful [%s]", migration.ID, time.Since(start)) + log.Infof("migrate: %s successful [%s]", migration.ID, time.Since(start)) } } } diff --git a/internal/migrate/mysql/20211121-094727.sql b/internal/migrate/mysql/20211121-094727.sql index 832c87417..1001fc8dc 100644 --- a/internal/migrate/mysql/20211121-094727.sql +++ b/internal/migrate/mysql/20211121-094727.sql @@ -1 +1 @@ -DROP INDEX IF EXISTS uix_places_place_label ON `places`; \ No newline at end of file +DROP INDEX uix_places_place_label ON `places`; \ No newline at end of file diff --git a/internal/migrate/mysql/20211124-120008.sql b/internal/migrate/mysql/20211124-120008.sql index 5cef86f6e..ec2134615 100644 --- a/internal/migrate/mysql/20211124-120008.sql +++ b/internal/migrate/mysql/20211124-120008.sql @@ -1,2 +1,2 @@ -DROP INDEX IF EXISTS idx_places_place_label ON `places`; -DROP INDEX IF EXISTS uix_places_label ON `places`; \ No newline at end of file +DROP INDEX idx_places_place_label ON `places`; +DROP INDEX uix_places_label ON `places`; \ No newline at end of file