From 3d908c7256b06d6ad5515aec3cb7b791bee0b4a7 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sun, 19 May 2024 14:17:01 +0200 Subject: [PATCH] =?UTF-8?q?Config:=20Add=20=E2=80=9Cdaily=E2=80=9D=20and?= =?UTF-8?q?=20=E2=80=9Cweekly=E2=80=9D=20backup=20schedule=20options=20#42?= =?UTF-8?q?43?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michael Mayer --- .../restore_albums.go => backup/albums.go} | 63 ++++++- internal/backup/albums_test.go | 35 ++++ internal/backup/backup.go | 31 ++++ internal/backup/backup_test.go | 29 ++++ internal/backup/const.go | 3 + .../database.go} | 141 +++++++++++++++- internal/backup/sync.go | 12 ++ internal/backup/testdata/.gitignore | 2 + internal/commands/backup.go | 6 +- internal/commands/restore.go | 6 +- internal/commands/start.go | 4 +- internal/config/config.go | 10 +- internal/config/config_backup.go | 13 +- internal/config/config_backup_test.go | 2 +- internal/config/flags.go | 4 +- internal/config/schedule.go | 41 +++++ internal/config/schedule_test.go | 23 +++ internal/photoprism/backup_albums.go | 70 -------- internal/photoprism/backup_database.go | 156 ------------------ internal/query/albums.go | 2 +- internal/workers/backup.go | 6 +- internal/workers/workers_test.go | 1 + 22 files changed, 394 insertions(+), 266 deletions(-) rename internal/{photoprism/restore_albums.go => backup/albums.go} (54%) create mode 100644 internal/backup/albums_test.go create mode 100644 internal/backup/backup.go create mode 100644 internal/backup/backup_test.go create mode 100644 internal/backup/const.go rename internal/{photoprism/restore_database.go => backup/database.go} (53%) create mode 100644 internal/backup/sync.go create mode 100644 internal/backup/testdata/.gitignore create mode 100644 internal/config/schedule.go create mode 100644 internal/config/schedule_test.go delete mode 100644 internal/photoprism/backup_albums.go delete mode 100644 internal/photoprism/backup_database.go diff --git a/internal/photoprism/restore_albums.go b/internal/backup/albums.go similarity index 54% rename from internal/photoprism/restore_albums.go rename to internal/backup/albums.go index 6a6d83816..ffb3b257d 100644 --- a/internal/photoprism/restore_albums.go +++ b/internal/backup/albums.go @@ -1,22 +1,81 @@ -package photoprism +package backup import ( "path/filepath" "regexp" + "time" "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/get" "github.com/photoprism/photoprism/internal/query" "github.com/photoprism/photoprism/pkg/clean" "github.com/photoprism/photoprism/pkg/fs" ) +// Albums creates a YAML file backup of all albums. +func Albums(backupPath string, force bool) (count int, err error) { + // Make sure only one backup/restore operation is running at a time. + backupAlbumsMutex.Lock() + defer backupAlbumsMutex.Unlock() + + // Get albums from database. + albums, queryErr := query.Albums(0, 1000000) + + if queryErr != nil { + return count, queryErr + } + + if !fs.PathExists(backupPath) { + backupPath = get.Config().BackupAlbumsPath() + } + + log.Debugf("backup: album backups will be stored in %s", clean.Log(backupPath)) + log.Infof("backup: saving album metadata in YAML backup files") + + var latest time.Time + + // Ignore the last modification timestamp if the force flag is set. + if !force { + latest = backupAlbumsTime + } + + // Save albums to YAML backup files. + for _, a := range albums { + // Album modification timestamp. + changed := a.UpdatedAt + + // Skip albums that have already been saved to YAML backup files. + if !force && !backupAlbumsTime.IsZero() && !changed.IsZero() && !backupAlbumsTime.Before(changed) { + continue + } + + // Remember the lastest modification timestamp. + if changed.After(latest) { + latest = changed + } + + // Write album metadata to YAML backup file. + if saveErr := a.SaveBackupYaml(backupPath); saveErr != nil { + err = saveErr + } else { + count++ + } + } + + // Set backupAlbumsTime to latest modification timestamp, + // so that already saved albums can be skipped next time. + backupAlbumsTime = latest + + return count, err +} + // RestoreAlbums restores all album YAML file backups. func RestoreAlbums(backupPath string, force bool) (count int, result error) { // Make sure only one backup/restore operation is running at a time. backupAlbumsMutex.Lock() defer backupAlbumsMutex.Unlock() - c := Config() + c := get.Config() if !c.BackupAlbums() && !force { log.Debugf("albums: metadata backup files are disabled") diff --git a/internal/backup/albums_test.go b/internal/backup/albums_test.go new file mode 100644 index 000000000..4bf56bc25 --- /dev/null +++ b/internal/backup/albums_test.go @@ -0,0 +1,35 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/photoprism/photoprism/pkg/fs" +) + +func TestAlbums(t *testing.T) { + backupPath, err := filepath.Abs("./testdata/albums") + + if err != nil { + t.Fatal(err) + } + + if err = os.MkdirAll(backupPath, fs.ModeDir); err != nil { + t.Fatal(err) + } + + count, err := Albums(backupPath, true) + + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 30, count) + + if err = os.RemoveAll(backupPath); err != nil { + t.Fatal(err) + } +} diff --git a/internal/backup/backup.go b/internal/backup/backup.go new file mode 100644 index 000000000..9647b9c22 --- /dev/null +++ b/internal/backup/backup.go @@ -0,0 +1,31 @@ +/* +Package backup provides backup and restore functions for databases and albums. + +Copyright (c) 2018 - 2024 PhotoPrism UG. All rights reserved. + + This program is free software: you can redistribute it and/or modify + it under Version 3 of the GNU Affero General Public License (the "AGPL"): + + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + The AGPL is supplemented by our Trademark and Brand Guidelines, + which describe how our Brand Assets may be used: + + +Feel free to send an email to hello@photoprism.app if you have questions, +want to support our work, or just want to say hello. + +Additional information can be found in our Developer Guide: + +*/ +package backup + +import ( + "github.com/photoprism/photoprism/internal/event" +) + +var log = event.Log diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 000000000..00316b0b5 --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,29 @@ +package backup + +import ( + "os" + "testing" + + "github.com/sirupsen/logrus" + + "github.com/photoprism/photoprism/internal/config" + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/internal/get" + "github.com/photoprism/photoprism/internal/photoprism" +) + +func TestMain(m *testing.M) { + log = logrus.StandardLogger() + log.SetLevel(logrus.TraceLevel) + event.AuditLog = log + + c := config.TestConfig() + defer c.CloseDb() + + get.SetConfig(c) + photoprism.SetConfig(c) + + code := m.Run() + + os.Exit(code) +} diff --git a/internal/backup/const.go b/internal/backup/const.go new file mode 100644 index 000000000..840edfeb8 --- /dev/null +++ b/internal/backup/const.go @@ -0,0 +1,3 @@ +package backup + +const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql" diff --git a/internal/photoprism/restore_database.go b/internal/backup/database.go similarity index 53% rename from internal/photoprism/restore_database.go rename to internal/backup/database.go index 8e3f43f13..927f03b3d 100644 --- a/internal/photoprism/restore_database.go +++ b/internal/backup/database.go @@ -1,4 +1,4 @@ -package photoprism +package backup import ( "bytes" @@ -11,14 +11,149 @@ import ( "regexp" "sort" "strings" + "time" + + "github.com/dustin/go-humanize/english" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/get" "github.com/photoprism/photoprism/pkg/clean" "github.com/photoprism/photoprism/pkg/fs" ) -const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql" +// Database creates a database backup dump with the specified file and path name. +func Database(backupPath, fileName string, toStdOut, force bool, retain int) (err error) { + // Ensure that only one database backup/restore operation is running at a time. + backupDatabaseMutex.Lock() + defer backupDatabaseMutex.Unlock() + + // Backup action shown in logs. + backupAction := "creating" + + // Get configuration. + c := get.Config() + + if !toStdOut { + if backupPath == "" { + backupPath = c.BackupDatabasePath() + } + + // Create the backup path if it does not already exist. + if err = fs.MkdirAll(backupPath); err != nil { + return err + } + + // Check if the backup path is writable. + if !fs.PathWritable(backupPath) { + return fmt.Errorf("backup path is not writable") + } + + if fileName == "" { + backupFile := time.Now().UTC().Format("2006-01-02") + ".sql" + fileName = filepath.Join(backupPath, backupFile) + } + + log.Debugf("backup: database backups will be stored in %s", clean.Log(backupPath)) + + if _, err = os.Stat(fileName); err == nil && !force { + return fmt.Errorf("%s already exists", clean.Log(filepath.Base(fileName))) + } else if err == nil { + backupAction = "replacing" + } + + // Create backup path if not exists. + if dir := filepath.Dir(fileName); dir != "." { + if err = fs.MkdirAll(dir); err != nil { + return err + } + } + } + + var cmd *exec.Cmd + + switch c.DatabaseDriver() { + case config.MySQL, config.MariaDB: + cmd = exec.Command( + c.MariadbDumpBin(), + "--protocol", "tcp", + "-h", c.DatabaseHost(), + "-P", c.DatabasePortString(), + "-u", c.DatabaseUser(), + "-p"+c.DatabasePassword(), + c.DatabaseName(), + ) + case config.SQLite3: + if !fs.FileExistsNotEmpty(c.DatabaseFile()) { + return fmt.Errorf("sqlite database file %s not found", clean.LogQuote(c.DatabaseFile())) + } + + cmd = exec.Command( + c.SqliteBin(), + c.DatabaseFile(), + ".dump", + ) + default: + return fmt.Errorf("unsupported database type: %s", c.DatabaseDriver()) + } + + // Write to stdout or file. + var f *os.File + if toStdOut { + log.Infof("backup: sending database backup to stdout") + f = os.Stdout + } else if f, err = os.OpenFile(fileName, os.O_TRUNC|os.O_RDWR|os.O_CREATE, fs.ModeBackup); err != nil { + return fmt.Errorf("failed to create %s (%s)", clean.Log(fileName), err) + } else { + log.Infof("backup: %s database backup file %s", backupAction, clean.Log(filepath.Base(fileName))) + defer f.Close() + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = f + + // Log exact command for debugging in trace mode. + log.Trace(cmd.String()) + + // Run backup command. + if cmdErr := cmd.Run(); cmdErr != nil { + if errStr := strings.TrimSpace(stderr.String()); errStr != "" { + return errors.New(errStr) + } + + return cmdErr + } + + // Delete old backups if the number of backup files to keep has been specified. + if !toStdOut && backupPath != "" && retain > 0 { + files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern)) + + if globErr != nil { + return globErr + } + + if len(files) == 0 { + return fmt.Errorf("found no database backup files in %s", backupPath) + } else if len(files) <= retain { + return nil + } + + sort.Strings(files) + + log.Infof("backup: retaining %s", english.Plural(retain, "database backup", "database backups")) + + for i := 0; i < len(files)-retain; i++ { + if err = os.Remove(files[i]); err != nil { + return err + } else { + log.Infof("backup: removed database backup file %s", clean.Log(filepath.Base(files[i]))) + } + } + } + + return nil +} // RestoreDatabase restores the database from a backup file with the specified path and name. func RestoreDatabase(backupPath, fileName string, fromStdIn, force bool) (err error) { @@ -26,7 +161,7 @@ func RestoreDatabase(backupPath, fileName string, fromStdIn, force bool) (err er backupDatabaseMutex.Lock() defer backupDatabaseMutex.Unlock() - c := Config() + c := get.Config() // If empty, use default backup file name. if !fromStdIn { diff --git a/internal/backup/sync.go b/internal/backup/sync.go new file mode 100644 index 000000000..6c2d69f24 --- /dev/null +++ b/internal/backup/sync.go @@ -0,0 +1,12 @@ +package backup + +import ( + "sync" + "time" +) + +var ( + backupDatabaseMutex = sync.Mutex{} + backupAlbumsMutex = sync.Mutex{} + backupAlbumsTime = time.Time{} +) diff --git a/internal/backup/testdata/.gitignore b/internal/backup/testdata/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/internal/backup/testdata/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/internal/commands/backup.go b/internal/commands/backup.go index aa3d39143..1d6ccab89 100644 --- a/internal/commands/backup.go +++ b/internal/commands/backup.go @@ -9,8 +9,8 @@ import ( "github.com/dustin/go-humanize/english" "github.com/urfave/cli" + "github.com/photoprism/photoprism/internal/backup" "github.com/photoprism/photoprism/internal/config" - "github.com/photoprism/photoprism/internal/photoprism" "github.com/photoprism/photoprism/pkg/fs" ) @@ -101,7 +101,7 @@ func backupAction(ctx *cli.Context) error { fileName = filepath.Join(databasePath, backupFile) } - if err = photoprism.BackupDatabase(databasePath, fileName, fileName == "-", force, retain); err != nil { + if err = backup.Database(databasePath, fileName, fileName == "-", force, retain); err != nil { return fmt.Errorf("failed to create database backup: %w", err) } } @@ -115,7 +115,7 @@ func backupAction(ctx *cli.Context) error { albumsPath = conf.BackupAlbumsPath() } - if count, backupErr := photoprism.BackupAlbums(albumsPath, true); backupErr != nil { + if count, backupErr := backup.Albums(albumsPath, true); backupErr != nil { return backupErr } else { log.Infof("backup: saved %s", english.Plural(count, "album backup", "album backups")) diff --git a/internal/commands/restore.go b/internal/commands/restore.go index b6c395238..db2476590 100644 --- a/internal/commands/restore.go +++ b/internal/commands/restore.go @@ -7,8 +7,8 @@ import ( "github.com/dustin/go-humanize/english" "github.com/urfave/cli" + "github.com/photoprism/photoprism/internal/backup" "github.com/photoprism/photoprism/internal/get" - "github.com/photoprism/photoprism/internal/photoprism" "github.com/photoprism/photoprism/pkg/clean" "github.com/photoprism/photoprism/pkg/fs" ) @@ -81,7 +81,7 @@ func restoreAction(ctx *cli.Context) error { // Restore database from backup dump? if !restoreDatabase { // Do nothing. - } else if err = photoprism.RestoreDatabase(databasePath, databaseFile, databaseFile == "-", force); err != nil { + } else if err = backup.RestoreDatabase(databasePath, databaseFile, databaseFile == "-", force); err != nil { return err } @@ -102,7 +102,7 @@ func restoreAction(ctx *cli.Context) error { } else { log.Infof("restore: restoring album backups from %s", clean.Log(albumsPath)) - if count, restoreErr := photoprism.RestoreAlbums(albumsPath, true); restoreErr != nil { + if count, restoreErr := backup.RestoreAlbums(albumsPath, true); restoreErr != nil { return restoreErr } else { log.Infof("restore: restored %s from YAML files", english.Plural(count, "album", "albums")) diff --git a/internal/commands/start.go b/internal/commands/start.go index e7c406670..0eee9121d 100644 --- a/internal/commands/start.go +++ b/internal/commands/start.go @@ -14,8 +14,8 @@ import ( "github.com/urfave/cli" "github.com/photoprism/photoprism/internal/auto" + "github.com/photoprism/photoprism/internal/backup" "github.com/photoprism/photoprism/internal/mutex" - "github.com/photoprism/photoprism/internal/photoprism" "github.com/photoprism/photoprism/internal/server" "github.com/photoprism/photoprism/internal/session" "github.com/photoprism/photoprism/internal/workers" @@ -126,7 +126,7 @@ func startAction(ctx *cli.Context) error { go server.Start(cctx, conf) // Restore albums from YAML files. - if count, restoreErr := photoprism.RestoreAlbums(conf.BackupAlbumsPath(), false); restoreErr != nil { + if count, restoreErr := backup.RestoreAlbums(conf.BackupAlbumsPath(), false); restoreErr != nil { log.Errorf("restore: %s (albums)", restoreErr) } else if count > 0 { log.Infof("restore: %s restored", english.Plural(count, "album backup", "album backups")) diff --git a/internal/config/config.go b/internal/config/config.go index 070eb4863..5f80f44b0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,7 +42,6 @@ import ( _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/klauspost/cpuid/v2" "github.com/pbnjay/memory" - "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -766,14 +765,7 @@ func (c *Config) IndexWorkers() int { // IndexSchedule returns the indexing schedule in cron format, e.g. "0 */3 * * *" to start indexing every 3 hours. func (c *Config) IndexSchedule() string { - if c.options.IndexSchedule == "" { - return "" - } else if _, err := cron.ParseStandard(c.options.IndexSchedule); err != nil { - log.Tracef("config: invalid auto indexing schedule (%s)", err) - return "" - } - - return c.options.IndexSchedule + return Schedule(c.options.IndexSchedule) } // WakeupInterval returns the duration between background worker runs diff --git a/internal/config/config_backup.go b/internal/config/config_backup.go index 044ab590e..bcabd0dc5 100644 --- a/internal/config/config_backup.go +++ b/internal/config/config_backup.go @@ -3,14 +3,12 @@ package config import ( "path/filepath" - "github.com/robfig/cron/v3" - "github.com/photoprism/photoprism/pkg/clean" "github.com/photoprism/photoprism/pkg/fs" ) const ( - DefaultBackupSchedule = "0 12 * * *" + DefaultBackupSchedule = "daily" DefaultBackupRetain = 3 ) @@ -34,14 +32,7 @@ func (c *Config) BackupBasePath() string { // BackupSchedule returns the backup schedule in cron format, e.g. "0 12 * * *" for daily at noon. func (c *Config) BackupSchedule() string { - if c.options.BackupSchedule == "" { - return "" - } else if _, err := cron.ParseStandard(c.options.BackupSchedule); err != nil { - log.Tracef("config: invalid backup schedule (%s)", err) - return "" - } - - return c.options.BackupSchedule + return Schedule(c.options.BackupSchedule) } // BackupRetain returns the maximum number of SQL database dumps to keep, or -1 to keep all. diff --git a/internal/config/config_backup_test.go b/internal/config/config_backup_test.go index 41b71fb2b..19f7bd3d7 100644 --- a/internal/config/config_backup_test.go +++ b/internal/config/config_backup_test.go @@ -18,7 +18,7 @@ func TestConfig_BackupBasePath(t *testing.T) { func TestConfig_BackupSchedule(t *testing.T) { c := NewConfig(CliTestContext()) - assert.Equal(t, DefaultBackupSchedule, c.BackupSchedule()) + assert.Contains(t, c.BackupSchedule(), " * * *") } func TestConfig_BackupRetain(t *testing.T) { diff --git a/internal/config/flags.go b/internal/config/flags.go index 628a174e0..910108f60 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -193,7 +193,7 @@ var Flags = CliFlags{ }}, { Flag: cli.StringFlag{ Name: "backup-schedule", - Usage: "backup `SCHEDULE` in cron format, e.g. \"0 12 * * *\" for daily at noon", + Usage: "backup `SCHEDULE` in cron format (e.g. \"0 12 * * *\" for daily at noon) or at a random time (daily, weekly)", Value: DefaultBackupSchedule, EnvVar: EnvVar("BACKUP_SCHEDULE"), }}, { @@ -221,7 +221,7 @@ var Flags = CliFlags{ }}, { Flag: cli.StringFlag{ Name: "index-schedule", - Usage: "indexing `SCHEDULE` in cron format, e.g. \"0 */3 * * *\" for every 3 hours (leave empty to disable)", + Usage: "regular indexing `SCHEDULE` in cron format (e.g. \"0 */3 * * *\" or \"@every 3h\" for every 3 hours; leave empty to disable)", Value: DefaultIndexSchedule, EnvVar: EnvVar("INDEX_SCHEDULE"), }}, { diff --git a/internal/config/schedule.go b/internal/config/schedule.go new file mode 100644 index 000000000..8d4623b16 --- /dev/null +++ b/internal/config/schedule.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "math/rand/v2" + "strings" + + "github.com/robfig/cron/v3" + + "github.com/photoprism/photoprism/pkg/clean" +) + +const ( + ScheduleDaily = "daily" + ScheduleWeekly = "weekly" +) + +// Schedule evaluates a schedule config value and returns it, or an empty string if it is invalid. Cron schedules consist +// of 5 space separated values: minute, hour, day of month, month and day of week, e.g. "0 12 * * *" for daily at noon. +func Schedule(s string) string { + if s == "" { + return "" + } + + s = strings.TrimSpace(strings.ToLower(s)) + + switch s { + case ScheduleDaily: + return fmt.Sprintf("%d %d * * *", rand.IntN(60), rand.IntN(24)) + case ScheduleWeekly: + return fmt.Sprintf("%d %d * * 0", rand.IntN(60), rand.IntN(24)) + } + + // Example: "0 12 * * *" stands for daily at noon. + if _, err := cron.ParseStandard(s); err != nil { + log.Warnf("config: invalid schedule %s (%s)", clean.Log(s), err) + return "" + } + + return s +} diff --git a/internal/config/schedule_test.go b/internal/config/schedule_test.go new file mode 100644 index 000000000..3cfb022b7 --- /dev/null +++ b/internal/config/schedule_test.go @@ -0,0 +1,23 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSchedule(t *testing.T) { + assert.Equal(t, "", Schedule("")) + assert.Equal(t, DefaultIndexSchedule, Schedule(DefaultIndexSchedule)) + + // Random default backup schedule. + backupSchedule := Schedule(DefaultBackupSchedule) + assert.Equal(t, backupSchedule, Schedule(backupSchedule)) + + // Regular backups at a random time (daily or weekly). + daily := Schedule(ScheduleDaily) + weekly := Schedule(ScheduleWeekly) + + assert.Equal(t, daily, Schedule(daily)) + assert.Equal(t, weekly, Schedule(weekly)) +} diff --git a/internal/photoprism/backup_albums.go b/internal/photoprism/backup_albums.go deleted file mode 100644 index 4e31ea58d..000000000 --- a/internal/photoprism/backup_albums.go +++ /dev/null @@ -1,70 +0,0 @@ -package photoprism - -import ( - "sync" - "time" - - "github.com/photoprism/photoprism/internal/query" - "github.com/photoprism/photoprism/pkg/clean" - "github.com/photoprism/photoprism/pkg/fs" -) - -var backupAlbumsTime = time.Time{} -var backupAlbumsMutex = sync.Mutex{} - -// BackupAlbums creates a YAML file backup of all albums. -func BackupAlbums(backupPath string, force bool) (count int, err error) { - // Make sure only one backup/restore operation is running at a time. - backupAlbumsMutex.Lock() - defer backupAlbumsMutex.Unlock() - - // Get albums from database. - albums, queryErr := query.Albums(0, 1000000) - - if queryErr != nil { - return count, queryErr - } - - if !fs.PathExists(backupPath) { - backupPath = Config().BackupAlbumsPath() - } - - log.Debugf("backup: album backups will be stored in %s", clean.Log(backupPath)) - log.Infof("backup: saving album metadata in YAML backup files") - - var latest time.Time - - // Ignore the last modification timestamp if the force flag is set. - if !force { - latest = backupAlbumsTime - } - - // Save albums to YAML backup files. - for _, a := range albums { - // Album modification timestamp. - changed := a.UpdatedAt - - // Skip albums that have already been saved to YAML backup files. - if !force && !backupAlbumsTime.IsZero() && !changed.IsZero() && !backupAlbumsTime.Before(changed) { - continue - } - - // Remember the lastest modification timestamp. - if changed.After(latest) { - latest = changed - } - - // Write album metadata to YAML backup file. - if saveErr := a.SaveBackupYaml(backupPath); saveErr != nil { - err = saveErr - } else { - count++ - } - } - - // Set backupAlbumsTime to latest modification timestamp, - // so that already saved albums can be skipped next time. - backupAlbumsTime = latest - - return count, err -} diff --git a/internal/photoprism/backup_database.go b/internal/photoprism/backup_database.go deleted file mode 100644 index dcbcc7f2c..000000000 --- a/internal/photoprism/backup_database.go +++ /dev/null @@ -1,156 +0,0 @@ -package photoprism - -import ( - "bytes" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" - "sort" - "strings" - "sync" - "time" - - "github.com/dustin/go-humanize/english" - - "github.com/photoprism/photoprism/internal/config" - "github.com/photoprism/photoprism/pkg/clean" - "github.com/photoprism/photoprism/pkg/fs" -) - -var backupDatabaseMutex = sync.Mutex{} - -// BackupDatabase creates a database backup dump with the specified file and path name. -func BackupDatabase(backupPath, fileName string, toStdOut, force bool, retain int) (err error) { - // Ensure that only one database backup/restore operation is running at a time. - backupDatabaseMutex.Lock() - defer backupDatabaseMutex.Unlock() - - // Backup action shown in logs. - backupAction := "creating" - - // Get configuration. - c := Config() - - if !toStdOut { - if backupPath == "" { - backupPath = c.BackupDatabasePath() - } - - // Create the backup path if it does not already exist. - if err = fs.MkdirAll(backupPath); err != nil { - return err - } - - // Check if the backup path is writable. - if !fs.PathWritable(backupPath) { - return fmt.Errorf("backup path is not writable") - } - - if fileName == "" { - backupFile := time.Now().UTC().Format("2006-01-02") + ".sql" - fileName = filepath.Join(backupPath, backupFile) - } - - log.Debugf("backup: database backups will be stored in %s", clean.Log(backupPath)) - - if _, err = os.Stat(fileName); err == nil && !force { - return fmt.Errorf("%s already exists", clean.Log(filepath.Base(fileName))) - } else if err == nil { - backupAction = "replacing" - } - - // Create backup path if not exists. - if dir := filepath.Dir(fileName); dir != "." { - if err = fs.MkdirAll(dir); err != nil { - return err - } - } - } - - var cmd *exec.Cmd - - switch c.DatabaseDriver() { - case config.MySQL, config.MariaDB: - cmd = exec.Command( - c.MariadbDumpBin(), - "--protocol", "tcp", - "-h", c.DatabaseHost(), - "-P", c.DatabasePortString(), - "-u", c.DatabaseUser(), - "-p"+c.DatabasePassword(), - c.DatabaseName(), - ) - case config.SQLite3: - if !fs.FileExistsNotEmpty(c.DatabaseFile()) { - return fmt.Errorf("sqlite database file %s not found", clean.LogQuote(c.DatabaseFile())) - } - - cmd = exec.Command( - c.SqliteBin(), - c.DatabaseFile(), - ".dump", - ) - default: - return fmt.Errorf("unsupported database type: %s", c.DatabaseDriver()) - } - - // Write to stdout or file. - var f *os.File - if toStdOut { - log.Infof("backup: sending database backup to stdout") - f = os.Stdout - } else if f, err = os.OpenFile(fileName, os.O_TRUNC|os.O_RDWR|os.O_CREATE, fs.ModeBackup); err != nil { - return fmt.Errorf("failed to create %s (%s)", clean.Log(fileName), err) - } else { - log.Infof("backup: %s database backup file %s", backupAction, clean.Log(filepath.Base(fileName))) - defer f.Close() - } - - var stderr bytes.Buffer - cmd.Stderr = &stderr - cmd.Stdout = f - - // Log exact command for debugging in trace mode. - log.Trace(cmd.String()) - - // Run backup command. - if cmdErr := cmd.Run(); cmdErr != nil { - if errStr := strings.TrimSpace(stderr.String()); errStr != "" { - return errors.New(errStr) - } - - return cmdErr - } - - // Delete old backups if the number of backup files to keep has been specified. - if !toStdOut && backupPath != "" && retain > 0 { - files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern)) - - if globErr != nil { - return globErr - } - - if len(files) == 0 { - return fmt.Errorf("found no database backup files in %s", backupPath) - } else if len(files) <= retain { - return nil - } - - sort.Strings(files) - - log.Infof("backup: retaining %s", english.Plural(retain, "database backup", "database backups")) - - for i := 0; i < len(files)-retain; i++ { - if err = os.Remove(files[i]); err != nil { - return err - } else { - log.Infof("backup: removed database backup file %s", clean.Log(filepath.Base(files[i]))) - } - } - } - - return nil -} diff --git a/internal/query/albums.go b/internal/query/albums.go index 864d9fbce..89f2d0e86 100644 --- a/internal/query/albums.go +++ b/internal/query/albums.go @@ -16,7 +16,7 @@ import ( // Albums returns a slice of albums. func Albums(offset, limit int) (results entity.Albums, err error) { - err = UnscopedDb().Table("albums").Select("*").Offset(offset).Limit(limit).Find(&results).Error + err = UnscopedDb().Table("albums").Select("*").Order("album_type, album_uid").Offset(offset).Limit(limit).Find(&results).Error return results, err } diff --git a/internal/workers/backup.go b/internal/workers/backup.go index a0b56e7a5..d683cb8e9 100644 --- a/internal/workers/backup.go +++ b/internal/workers/backup.go @@ -8,9 +8,9 @@ import ( "github.com/dustin/go-humanize/english" + "github.com/photoprism/photoprism/internal/backup" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/mutex" - "github.com/photoprism/photoprism/internal/photoprism" ) // Backup represents a background backup worker. @@ -58,7 +58,7 @@ func (w *Backup) Start(database, albums bool, force bool, retain int) (err error if database { databasePath := w.conf.BackupDatabasePath() - if err = photoprism.BackupDatabase(databasePath, "", false, force, retain); err != nil { + if err = backup.Database(databasePath, "", false, force, retain); err != nil { log.Errorf("backup: %s (database)", err) } } @@ -71,7 +71,7 @@ func (w *Backup) Start(database, albums bool, force bool, retain int) (err error if albums { albumsPath := w.conf.BackupAlbumsPath() - if count, backupErr := photoprism.BackupAlbums(albumsPath, false); backupErr != nil { + if count, backupErr := backup.Albums(albumsPath, false); backupErr != nil { log.Errorf("backup: %s (albums)", backupErr.Error()) } else if count > 0 { log.Infof("backup: saved %s", english.Plural(count, "album backup", "album backups")) diff --git a/internal/workers/workers_test.go b/internal/workers/workers_test.go index 5cd823c75..6b4a2305d 100644 --- a/internal/workers/workers_test.go +++ b/internal/workers/workers_test.go @@ -19,6 +19,7 @@ func TestMain(m *testing.M) { c := config.TestConfig() defer c.CloseDb() + get.SetConfig(c) photoprism.SetConfig(c)