MySQL 8: Improve migrate command, ignore errors when dropping indexes

docker-compose.ci.yml

@@ -38,6 +38,7 @@ services:
       PHOTOPRISM_DATABASE_PASSWORD: "photoprism"
       PHOTOPRISM_TEST_DRIVER: "sqlite"
       PHOTOPRISM_TEST_DSN: ".test.db"
+      PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true"
       PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters)
       PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets"
       PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage"
@@ -157,6 +158,21 @@ services:
       MYSQL_PASSWORD: photoprism
       MYSQL_DATABASE: photoprism
 
+  ## MySQL Database Server
+  ## Docs: https://dev.mysql.com/doc/refman/8.0/en/
+  mysql:
+    image: mysql:8
+    command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
+    expose:
+      - "4001" # Database port (internal)
+    volumes:
+      - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
+    environment:
+      MYSQL_ROOT_PASSWORD: photoprism
+      MYSQL_USER: photoprism
+      MYSQL_PASSWORD: photoprism
+      MYSQL_DATABASE: photoprism
+
   ## Dummy WebDAV Server
   dummy-webdav:
     image: photoprism/dummy-webdav:20211109

docker-compose.mariadb.yml

@@ -57,19 +57,6 @@ services:
       MYSQL_PASSWORD: photoprism
       MYSQL_DATABASE: photoprism
 
-  mysql-8:
-    image: mysql:8
-    command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
-    expose:
-      - "4001" # Database port (internal)
-    volumes:
-      - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
-    environment:
-      MYSQL_ROOT_PASSWORD: photoprism
-      MYSQL_USER: photoprism
-      MYSQL_PASSWORD: photoprism
-      MYSQL_DATABASE: photoprism
-
 networks:
   default:
     external:

docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - "2343:2343" # Acceptance Test HTTP port (host:container)
     shm_size: "2gb"
     environment:
+      PHOTOPRISM_ADMIN_PASSWORD: "photoprism"        # The initial admin password (min 4 characters)
       PHOTOPRISM_UID: ${UID:-1000}
       PHOTOPRISM_GID: ${GID:-1000}
       PHOTOPRISM_SITE_URL: "http://localhost:2342/"
@@ -44,7 +45,7 @@ services:
       PHOTOPRISM_DATABASE_PASSWORD: "photoprism"
       PHOTOPRISM_TEST_DRIVER: "sqlite"
       PHOTOPRISM_TEST_DSN: ".test.db"
-      PHOTOPRISM_ADMIN_PASSWORD: "photoprism"        # The initial admin password (min 4 characters)
+      PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true"
       PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets"
       PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage"
       PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals"
@@ -103,6 +104,21 @@ services:
       MYSQL_PASSWORD: photoprism
       MYSQL_DATABASE: photoprism
 
+  ## MySQL Database Server
+  ## Docs: https://dev.mysql.com/doc/refman/8.0/en/
+  mysql:
+    image: mysql:8
+    command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
+    expose:
+      - "4001" # Database port (internal)
+    volumes:
+      - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
+    environment:
+      MYSQL_ROOT_PASSWORD: photoprism
+      MYSQL_USER: photoprism
+      MYSQL_PASSWORD: photoprism
+      MYSQL_DATABASE: photoprism
+
   ## Dummy WebDAV Server
   dummy-webdav:
     image: photoprism/dummy-webdav:20211109

internal/commands/migrate.go

@@ -4,15 +4,26 @@ import (
 	"context"
 	"time"
 
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 
 	"github.com/photoprism/photoprism/internal/config"
 )
 
-// MigrateCommand registers the migrate cli command.
+// MigrateCommand registers the "migrate" CLI command.
 var MigrateCommand = cli.Command{
 	Name:  "migrate",
 	Usage: "Updates the index database schema",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "failed, f",
+			Usage: "run previously failed migrations",
+		},
+		cli.BoolFlag{
+			Name:  "trace, t",
+			Usage: "show trace logs for debugging",
+		},
+	},
 	Action: migrateAction,
 }
 
@@ -29,15 +40,26 @@ func migrateAction(ctx *cli.Context) error {
 		return err
 	}
 
+	defer conf.Shutdown()
+
+	if ctx.Bool("trace") {
+		log.SetLevel(logrus.TraceLevel)
+		log.Infoln("migrate: enabled trace mode")
+	}
+
+	runFailed := ctx.Bool("failed")
+
+	if runFailed {
+		log.Infoln("migrate: running previously failed migrations")
+	}
+
 	log.Infoln("migrating database schema...")
 
-	conf.InitDb()
+	conf.MigrateDb(runFailed)
 
 	elapsed := time.Since(start)
 
 	log.Infof("migration completed in %s", elapsed)
 
-	conf.Shutdown()
-
 	return nil
 }

internal/commands/places.go

@@ -16,11 +16,18 @@ import (
 // PlacesCommand registers the places subcommands.
 var PlacesCommand = cli.Command{
 	Name:  "places",
-	Usage: "Geographic data subcommands",
+	Usage: "Maps and location details subcommands",
 	Subcommands: []cli.Command{
 		{
 			Name:  "update",
 			Usage: "Retrieves updated location details",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:   "yes, y",
+					Hidden: true,
+					Usage:  "assume \"yes\" as answer to all prompts and run non-interactively",
+				},
+			},
 			Action: placesUpdateAction,
 		},
 	},
@@ -46,6 +53,7 @@ func placesUpdateAction(ctx *cli.Context) error {
 		return nil
 	}
 
+	if !ctx.Bool("yes") {
 		confirmPrompt := promptui.Prompt{
 			Label:     "Interrupting the update may result in inconsistent location details. Proceed?",
 			IsConfirm: true,
@@ -55,6 +63,7 @@ func placesUpdateAction(ctx *cli.Context) error {
 		if _, err := confirmPrompt.Run(); err != nil {
 			return nil
 		}
+	}
 
 	start := time.Now()
 

internal/commands/reset.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/manifoldco/promptui"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 
 	"github.com/photoprism/photoprism/internal/config"
@@ -19,18 +20,25 @@ import (
 var ResetCommand = cli.Command{
 	Name:  "reset",
 	Usage: "Resets the index and removes generated sidecar files",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "index, i",
+			Usage: "reset index database only",
+		},
+		cli.BoolFlag{
+			Name:  "trace, t",
+			Usage: "show trace logs for debugging",
+		},
+		cli.BoolFlag{
+			Name:  "yes, y",
+			Usage: "assume \"yes\" as answer to all prompts and run non-interactively",
+		},
+	},
 	Action: resetAction,
 }
 
 // resetAction resets the index and removes sidecar files after confirmation.
 func resetAction(ctx *cli.Context) error {
-	log.Warnf("YOU ARE ABOUT TO RESET THE INDEX AND REMOVE ALL JSON / YAML SIDECAR FILES")
-
-	removeIndexPrompt := promptui.Prompt{
-		Label:     "Reset index database including albums and metadata?",
-		IsConfirm: true,
-	}
-
 	conf := config.NewConfig(ctx)
 	_, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -39,101 +47,73 @@ func resetAction(ctx *cli.Context) error {
 		return err
 	}
 
+	defer conf.Shutdown()
+
 	entity.SetDbProvider(conf)
 
-	if _, err := removeIndexPrompt.Run(); err == nil {
+	if !ctx.Bool("yes") {
-		start := time.Now()
+		log.Warnf("This will delete and recreate your index database after confirmation")
 
-		tables := entity.Entities
+		if !ctx.Bool("index") {
-
+			log.Warnf("You will be asked next if you also want to remove all JSON and YAML backup files")
-		log.Infoln("dropping existing tables")
+		}
-		tables.Drop()
-
-		log.Infoln("restoring default schema")
-		entity.MigrateDb(true)
-
-		if conf.AdminPassword() != "" {
-			log.Infoln("restoring initial admin password")
-			entity.Admin.InitPassword(conf.AdminPassword())
 	}
 
-		log.Infof("database reset completed in %s", time.Since(start))
+	if ctx.Bool("trace") {
+		log.SetLevel(logrus.TraceLevel)
+		log.Infoln("reset: enabled trace mode")
+	}
+
+	resetIndex := ctx.Bool("yes")
+
+	// Show prompt?
+	if !resetIndex {
+		removeIndexPrompt := promptui.Prompt{
+			Label:     "Delete and recreate index database?",
+			IsConfirm: true,
+		}
+
+		if _, err := removeIndexPrompt.Run(); err == nil {
+			resetIndex = true
 		} else {
 			log.Infof("keeping index database")
 		}
+	}
+
+	// Reset index?
+	if resetIndex {
+		resetIndexDb(conf)
+	}
+
+	// Reset index only?
+	if ctx.Bool("index") {
+		return nil
+	}
 
 	removeSidecarJsonPrompt := promptui.Prompt{
-		Label:     "Permanently delete existing JSON metadata sidecar files?",
+		Label:     "Delete all JSON metadata sidecar files?",
 		IsConfirm: true,
 	}
 
 	if _, err := removeSidecarJsonPrompt.Run(); err == nil {
-		start := time.Now()
+		resetSidecarJson(conf)
-
-		matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json")
-
-		if err != nil {
-			return err
-		}
-
-		if len(matches) > 0 {
-			log.Infof("removing %d JSON metadata sidecar files", len(matches))
-
-			for _, name := range matches {
-				if err := os.Remove(name); err != nil {
-					fmt.Print("E")
-				} else {
-					fmt.Print(".")
-				}
-			}
-
-			fmt.Println("")
-
-			log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start))
-		} else {
-			log.Infof("found no JSON metadata sidecar files")
-		}
 	} else {
 		log.Infof("keeping JSON metadata sidecar files")
 	}
 
 	removeSidecarYamlPrompt := promptui.Prompt{
-		Label:     "Permanently delete existing YAML metadata backups?",
+		Label:     "Delete all YAML metadata backup files?",
 		IsConfirm: true,
 	}
 
 	if _, err := removeSidecarYamlPrompt.Run(); err == nil {
-		start := time.Now()
+		resetSidecarYaml(conf)
-
-		matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml")
-
-		if err != nil {
-			return err
-		}
-
-		if len(matches) > 0 {
-			log.Infof("%d YAML metadata backups will be removed", len(matches))
-
-			for _, name := range matches {
-				if err := os.Remove(name); err != nil {
-					fmt.Print("E")
-				} else {
-					fmt.Print(".")
-				}
-			}
-
-			fmt.Println("")
-
-			log.Infof("removed all YAML metadata backups [%s]", time.Since(start))
-		} else {
-			log.Infof("found no YAML metadata backups")
-		}
 	} else {
 		log.Infof("keeping YAML metadata backups")
 	}
 
 	removeAlbumYamlPrompt := promptui.Prompt{
-		Label:     "Permanently delete existing YAML album backups?",
+		Label:     "Delete all YAML album backup files?",
 		IsConfirm: true,
 	}
 
@@ -167,7 +147,85 @@ func resetAction(ctx *cli.Context) error {
 		log.Infof("keeping YAML album backup files")
 	}
 
-	conf.Shutdown()
-
 	return nil
 }
+
+// resetIndexDb resets the index database schema.
+func resetIndexDb(conf *config.Config) {
+	start := time.Now()
+
+	tables := entity.Entities
+
+	log.Infoln("dropping existing tables")
+	tables.Drop(conf.Db())
+
+	log.Infoln("restoring default schema")
+	entity.MigrateDb(true, false)
+
+	if conf.AdminPassword() != "" {
+		log.Infoln("restoring initial admin password")
+		entity.Admin.InitPassword(conf.AdminPassword())
+	}
+
+	log.Infof("database reset completed in %s", time.Since(start))
+}
+
+// resetSidecarJson removes generated JSON sidecar files.
+func resetSidecarJson(conf *config.Config) {
+	start := time.Now()
+
+	matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json")
+
+	if err != nil {
+		log.Errorf("reset: %s (find json sidecar files)", err)
+		return
+	}
+
+	if len(matches) > 0 {
+		log.Infof("removing %d JSON metadata sidecar files", len(matches))
+
+		for _, name := range matches {
+			if err := os.Remove(name); err != nil {
+				fmt.Print("E")
+			} else {
+				fmt.Print(".")
+			}
+		}
+
+		fmt.Println("")
+
+		log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start))
+	} else {
+		log.Infof("found no JSON metadata sidecar files")
+	}
+}
+
+// resetSidecarYaml removes generated YAML sidecar files.
+func resetSidecarYaml(conf *config.Config) {
+	start := time.Now()
+
+	matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml")
+
+	if err != nil {
+		log.Errorf("reset: %s (find yaml sidecar files)", err)
+		return
+	}
+
+	if len(matches) > 0 {
+		log.Infof("%d YAML metadata backups will be removed", len(matches))
+
+		for _, name := range matches {
+			if err := os.Remove(name); err != nil {
+				fmt.Print("E")
+			} else {
+				fmt.Print(".")
+			}
+		}
+
+		fmt.Println("")
+
+		log.Infof("removed all YAML metadata backups [%s]", time.Since(start))
+	} else {
+		log.Infof("found no YAML metadata backups")
+	}
+}

internal/commands/restore.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize/english"
-
 	"github.com/urfave/cli"
 
 	"github.com/photoprism/photoprism/internal/config"
@@ -151,7 +150,7 @@ func restoreAction(ctx *cli.Context) error {
 			)
 		case config.SQLite:
 			log.Infoln("dropping existing tables")
-			tables.Drop()
+			tables.Drop(conf.Db())
 			cmd = exec.Command(
 				conf.SqliteBin(),
 				conf.DatabaseDsn(),

internal/config/db.go

@@ -232,11 +232,16 @@ func (c *Config) SetDbOptions() {
 	}
 }
 
-// InitDb will initialize the database connection and schema.
+// InitDb initializes the database without running previously failed migrations.
 func (c *Config) InitDb() {
+	c.MigrateDb(false)
+}
+
+// MigrateDb initializes the database and migrates the schema if needed.
+func (c *Config) MigrateDb(runFailed bool) {
 	c.SetDbOptions()
 	entity.SetDbProvider(c)
-	entity.MigrateDb(false)
+	entity.MigrateDb(true, runFailed)
 
 	entity.Admin.InitPassword(c.AdminPassword())
 

internal/entity/db_fixtures.go

@@ -13,9 +13,9 @@ func CreateDefaultFixtures() {
 
 // ResetTestFixtures re-creates registered database tables and inserts test fixtures.
 func ResetTestFixtures() {
-	Entities.Migrate()
+	Entities.Migrate(Db(), false)
-	Entities.WaitForMigration()
+	Entities.WaitForMigration(Db())
-	Entities.Truncate()
+	Entities.Truncate(Db())
 
 	CreateDefaultFixtures()
 

internal/entity/db_migrate.go

@@ -1,13 +1,13 @@
 package entity
 
 // MigrateDb creates database tables and inserts default fixtures as needed.
-func MigrateDb(dropDeprecated bool) {
+func MigrateDb(dropDeprecated, runFailed bool) {
 	if dropDeprecated {
-		DeprecatedTables.Drop()
+		DeprecatedTables.Drop(Db())
 	}
 
-	Entities.Migrate()
+	Entities.Migrate(Db(), runFailed)
-	Entities.WaitForMigration()
+	Entities.WaitForMigration(Db())
 
 	CreateDefaultFixtures()
 }

internal/entity/db_mysql8_test.go

@@ -0,0 +1,51 @@
+package entity
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/jinzhu/gorm"
+)
+
+func TestMySQL8(t *testing.T) {
+	dbDriver := MySQL
+	dbDsn := os.Getenv("PHOTOPRISM_TEST_DSN_MYSQL8")
+
+	db, err := gorm.Open(dbDriver, dbDsn)
+
+	if err != nil || db == nil {
+		for i := 1; i <= 5; i++ {
+			db, err = gorm.Open(dbDriver, dbDsn)
+
+			if db != nil && err == nil {
+				break
+			}
+
+			time.Sleep(5 * time.Second)
+		}
+
+		if err != nil || db == nil {
+			t.Fatal(err)
+		}
+	}
+
+	defer db.Close()
+
+	db.LogMode(false)
+
+	DeprecatedTables.Drop(db)
+	Entities.Drop(db)
+
+	// First migration.
+	Entities.Migrate(db, false)
+	Entities.WaitForMigration(db)
+
+	// Second migration.
+	Entities.Migrate(db, false)
+	Entities.WaitForMigration(db)
+
+	// Third migration with force flag.
+	Entities.Migrate(db, true)
+	Entities.WaitForMigration(db)
+}

internal/entity/db_tables.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/jinzhu/gorm"
+
 	"github.com/photoprism/photoprism/internal/migrate"
 	"github.com/photoprism/photoprism/pkg/txt"
 )
@@ -44,7 +46,7 @@ var Entities = Tables{
 }
 
 // WaitForMigration waits for the database migration to be successful.
-func (list Tables) WaitForMigration() {
+func (list Tables) WaitForMigration(db *gorm.DB) {
 	type RowCount struct {
 		Count int
 	}
@@ -53,7 +55,7 @@ func (list Tables) WaitForMigration() {
 	for name := range list {
 		for i := 0; i <= attempts; i++ {
 			count := RowCount{}
-			if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
+			if err := db.Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
 				log.Tracef("entity: %s migrated", txt.Quote(name))
 				break
 			} else {
@@ -70,9 +72,9 @@ func (list Tables) WaitForMigration() {
 }
 
 // Truncate removes all data from tables without dropping them.
-func (list Tables) Truncate() {
+func (list Tables) Truncate(db *gorm.DB) {
 	for name := range list {
-		if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
+		if err := db.Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
 			// log.Debugf("entity: removed all data from %s", name)
 			break
 		} else if err.Error() != "record not found" {
@@ -82,29 +84,29 @@ func (list Tables) Truncate() {
 }
 
 // Migrate migrates all database tables of registered entities.
-func (list Tables) Migrate() {
+func (list Tables) Migrate(db *gorm.DB, runFailed bool) {
 	for name, entity := range list {
-		if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
+		if err := db.AutoMigrate(entity).Error; err != nil {
 			log.Debugf("entity: %s (waiting 1s)", err.Error())
 
 			time.Sleep(time.Second)
 
-			if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
+			if err := db.AutoMigrate(entity).Error; err != nil {
 				log.Errorf("entity: failed migrating %s", txt.Quote(name))
 				panic(err)
 			}
 		}
 	}
 
-	if err := migrate.Auto(Db()); err != nil {
+	if err := migrate.Auto(db, runFailed); err != nil {
 		log.Error(err)
 	}
 }
 
 // Drop drops all database tables of registered entities.
-func (list Tables) Drop() {
+func (list Tables) Drop(db *gorm.DB) {
 	for _, entity := range list {
-		if err := UnscopedDb().DropTableIfExists(entity).Error; err != nil {
+		if err := db.DropTableIfExists(entity).Error; err != nil {
 			panic(err)
 		}
 	}

internal/entity/deprecated.go

@@ -1,12 +1,14 @@
 package entity
 
+import "github.com/jinzhu/gorm"
+
 // Deprecated represents a list of deprecated database tables.
 type Deprecated []string
 
 // Drop drops all deprecated tables.
-func (list Deprecated) Drop() {
+func (list Deprecated) Drop(db *gorm.DB) {
 	for _, tableName := range list {
-		if err := UnscopedDb().DropTableIfExists(tableName).Error; err != nil {
+		if err := db.DropTableIfExists(tableName).Error; err != nil {
 			log.Debugf("drop %s: %s", tableName, err)
 		}
 	}

internal/migrate/auto.go

@@ -7,7 +7,7 @@ import (
 )
 
 // Auto automatically migrates the database provided.
-func Auto(db *gorm.DB) error {
+func Auto(db *gorm.DB, runFailed bool) error {
 	if db == nil {
 		return fmt.Errorf("migrate: database connection required")
 	}
@@ -23,7 +23,7 @@ func Auto(db *gorm.DB) error {
 	}
 
 	if migrations, ok := Dialects[name]; ok && len(migrations) > 0 {
-		migrations.Start(db)
+		migrations.Start(db, runFailed)
 		return nil
 	} else {
 		return fmt.Errorf("migrate: no migrations found for %s", name)

internal/migrate/dialect_mysql.go

@@ -5,11 +5,11 @@ var DialectMySQL = Migrations{
 	{
 		ID:         "20211121-094727",
 		Dialect:    "mysql",
-		Statements: []string{"DROP INDEX IF EXISTS uix_places_place_label ON `places`;"},
+		Statements: []string{"DROP INDEX uix_places_place_label ON `places`;"},
 	},
 	{
 		ID:         "20211124-120008",
 		Dialect:    "mysql",
-		Statements: []string{"DROP INDEX IF EXISTS idx_places_place_label ON `places`;", "DROP INDEX IF EXISTS uix_places_label ON `places`;"},
+		Statements: []string{"DROP INDEX idx_places_place_label ON `places`;", "DROP INDEX uix_places_label ON `places`;"},
 	},
 }

internal/migrate/migration.go

@@ -1,6 +1,7 @@
 package migrate
 
 import (
+	"strings"
 	"time"
 
 	"github.com/jinzhu/gorm"
@@ -29,6 +30,7 @@ func (m *Migration) Fail(err error, db *gorm.DB) {
 	}
 
 	m.Error = err.Error()
+
 	db.Model(m).Updates(Values{"Error": m.Error})
 }
 
@@ -41,9 +43,13 @@ func (m *Migration) Finish(db *gorm.DB) error {
 func (m *Migration) Execute(db *gorm.DB) error {
 	for _, s := range m.Statements {
 		if err := db.Exec(s).Error; err != nil {
+			if strings.HasPrefix(s, "DROP ") && strings.Contains(err.Error(), "DROP") {
+				log.Tracef("migrate: %s (drop statement)", err)
+			} else {
 				return err
 			}
 		}
+	}
 
 	return nil
 }

internal/migrate/migrations.go

@@ -1,33 +1,84 @@
 package migrate
 
 import (
+	"database/sql"
 	"time"
 
+	"github.com/dustin/go-humanize/english"
 	"github.com/jinzhu/gorm"
 )
 
 // Migrations represents a sorted list of migrations.
 type Migrations []Migration
 
+// MigrationMap represents a map of migrations.
+type MigrationMap map[string]Migration
+
+// Existing finds and returns previously executed database schema migrations.
+func Existing(db *gorm.DB) MigrationMap {
+	result := make(MigrationMap)
+	dialect := db.Dialect().GetName()
+
+	stmt := db.Model(Migration{}).Where("dialect = ?", dialect)
+	stmt = stmt.Select("id, dialect, error, source, started_at, finished_at")
+
+	rows, err := stmt.Rows()
+
+	if err != nil {
+		log.Warnf("migrate: %s (find existing)", err)
+		return result
+	}
+
+	defer func(rows *sql.Rows) {
+		err = rows.Close()
+	}(rows)
+
+	for rows.Next() {
+		m := Migration{}
+
+		if err = rows.Scan(&m.ID, &m.Dialect, &m.Error, &m.Source, &m.StartedAt, &m.FinishedAt); err != nil {
+			log.Warnf("migrate: %s (scan existing)", err)
+			return result
+		}
+
+		result[m.ID] = m
+	}
+
+	return result
+}
+
 // Start runs all migrations that haven't been executed yet.
-func (m *Migrations) Start(db *gorm.DB) {
+func (m *Migrations) Start(db *gorm.DB, runFailed bool) {
+	// Find previously executed migrations.
+	executed := Existing(db)
+
+	log.Debugf("migrate: found %s", english.Plural(len(executed), "previous migration", "previous migrations"))
+
 	for _, migration := range *m {
 		start := time.Now()
-
 		migration.StartedAt = start.UTC().Round(time.Second)
 
-		// Continue if already executed.
+		// Already executed?
-		if err := db.Create(migration).Error; err != nil {
+		if done, ok := executed[migration.ID]; ok {
+			// Try to run failed migrations again?
+			if !runFailed || done.Error == "" {
+				log.Debugf("migrate: %s skipped", migration.ID)
+				continue
+			}
+		} else if err := db.Create(migration).Error; err != nil {
+			// Should not happen.
+			log.Warnf("migrate: creating %s failed with %s [%s]", migration.ID, err, time.Since(start))
 			continue
 		}
 
+		// Run migration.
 		if err := migration.Execute(db); err != nil {
 			migration.Fail(err, db)
-			log.Errorf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start))
+			log.Errorf("migrate: executing %s failed with %s [%s]", migration.ID, err, time.Since(start))
 		} else if err = migration.Finish(db); err != nil {
-			log.Warnf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start))
+			log.Warnf("migrate: updating %s failed with %s [%s]", migration.ID, err, time.Since(start))
 		} else {
-			log.Infof("migration %s successful [%s]", migration.ID, time.Since(start))
+			log.Infof("migrate: %s successful [%s]", migration.ID, time.Since(start))
 		}
 	}
 }

internal/migrate/mysql/20211121-094727.sql

@@ -1 +1 @@
-DROP INDEX IF EXISTS uix_places_place_label ON `places`;
+DROP INDEX uix_places_place_label ON `places`;

internal/migrate/mysql/20211124-120008.sql

@@ -1,2 +1,2 @@
-DROP INDEX IF EXISTS idx_places_place_label ON `places`;
+DROP INDEX idx_places_place_label ON `places`;
-DROP INDEX IF EXISTS uix_places_label ON `places`;
+DROP INDEX uix_places_label ON `places`;