!100 定时清理数据库备份数据

* feat: 优化数据库 BINLOG 同步机制
* feat: 删除数据库实例前需删除关联的数据库备份与恢复任务
* refactor: 重构数据库备份与恢复模块
* feat: 定时清理数据库备份历史和本地 Binlog 文件
* feat: 压缩数据库备份文件

server/internal/db/infrastructure/persistence/db_backup.go

@@ -64,7 +64,6 @@ func (d *dbBackupRepoImpl) ListToDo(jobs any) error {
 
 // GetPageList 分页获取数据库备份任务列表
 func (d *dbBackupRepoImpl) GetPageList(condition *entity.DbBackupQuery, pageParam *model.PageParam, toEntity any, _ ...string) (*model.PageResult[any], error) {
-	d.GetModel()
 	qd := gormx.NewQuery(d.GetModel()).
 		Eq("id", condition.Id).
 		Eq0("db_instance_id", condition.DbInstanceId).
@@ -83,12 +82,16 @@ func (d *dbBackupRepoImpl) UpdateEnabled(_ context.Context, jobId uint64, enable
 	cond := map[string]any{
 		"id": jobId,
 	}
-	desc := "任务已禁用"
+	desc := "已禁用"
 	if enabled {
-		desc = "任务已启用"
+		desc = "已启用"
 	}
 	return d.Updates(cond, map[string]any{
 		"enabled":      enabled,
 		"enabled_desc": desc,
 	})
 }
+
+func (d *dbBackupRepoImpl) ListByCond(cond any, listModels any, cols ...string) error {
+	return d.dbJobBaseImpl.ListByCond(cond, listModels, cols...)
+}

server/internal/db/infrastructure/persistence/db_backup_history.go

@@ -34,12 +34,13 @@ func (repo *dbBackupHistoryRepoImpl) GetPageList(condition *entity.DbBackupHisto
 func (repo *dbBackupHistoryRepoImpl) GetHistories(backupHistoryIds []uint64, toEntity any) error {
 	return global.Db.Model(repo.GetModel()).
 		Where("id in ?", backupHistoryIds).
+		Where("deleting = false").
 		Scopes(gormx.UndeleteScope).
 		Find(toEntity).
 		Error
 }
 
-func (repo *dbBackupHistoryRepoImpl) GetLatestHistory(instanceId uint64, dbName string, bi *entity.BinlogInfo) (*entity.DbBackupHistory, error) {
+func (repo *dbBackupHistoryRepoImpl) GetLatestHistoryForBinlog(instanceId uint64, dbName string, bi *entity.BinlogInfo) (*entity.DbBackupHistory, error) {
 	history := &entity.DbBackupHistory{}
 	db := global.Db
 	err := db.Model(repo.GetModel()).
@@ -48,6 +49,8 @@ func (repo *dbBackupHistoryRepoImpl) GetLatestHistory(instanceId uint64, dbName
 		Where(db.Where("binlog_sequence < ?", bi.Sequence).
 			Or(db.Where("binlog_sequence = ?", bi.Sequence).
 				Where("binlog_position <= ?", bi.Position))).
+		Where("binlog_sequence > 0").
+		Where("deleting = false").
 		Scopes(gormx.UndeleteScope).
 		Order("binlog_sequence desc, binlog_position desc").
 		First(history).Error
@@ -57,10 +60,12 @@ func (repo *dbBackupHistoryRepoImpl) GetLatestHistory(instanceId uint64, dbName
 	return history, err
 }
 
-func (repo *dbBackupHistoryRepoImpl) GetEarliestHistory(instanceId uint64) (*entity.DbBackupHistory, bool, error) {
+func (repo *dbBackupHistoryRepoImpl) GetEarliestHistoryForBinlog(instanceId uint64) (*entity.DbBackupHistory, bool, error) {
 	history := &entity.DbBackupHistory{}
 	db := global.Db.Model(repo.GetModel())
 	err := db.Where("db_instance_id = ?", instanceId).
+		Where("binlog_sequence > 0").
+		Where("deleting = false").
 		Scopes(gormx.UndeleteScope).
 		Order("binlog_sequence").
 		First(history).Error
@@ -79,7 +84,7 @@ func (repo *dbBackupHistoryRepoImpl) UpdateDeleting(deleting bool, backupHistory
 		Where("id in ?", backupHistoryId).
 		Where("restoring = false").
 		Scopes(gormx.UndeleteScope).
-		Update("restoring", deleting)
+		Update("deleting", deleting)
 	if db.Error != nil {
 		return false, db.Error
 	}
@@ -103,3 +108,15 @@ func (repo *dbBackupHistoryRepoImpl) UpdateRestoring(restoring bool, backupHisto
 	}
 	return true, nil
 }
+
+func (repo *dbBackupHistoryRepoImpl) ZeroBinlogInfo(backupHistoryId uint64) error {
+	return global.Db.Model(repo.GetModel()).
+		Where("id = ?", backupHistoryId).
+		Where("restoring = false").
+		Scopes(gormx.UndeleteScope).
+		Updates(&map[string]any{
+			"binlog_file_name": "",
+			"binlog_sequence":  0,
+			"binlog_position":  0,
+		}).Error
+}

server/internal/db/infrastructure/persistence/db_binlog_history.go

@@ -7,6 +7,7 @@ import (
 	"mayfly-go/internal/db/domain/entity"
 	"mayfly-go/internal/db/domain/repository"
 	"mayfly-go/pkg/base"
+	"mayfly-go/pkg/global"
 	"mayfly-go/pkg/gormx"
 	"time"
 )
@@ -82,7 +83,7 @@ func (repo *dbBinlogHistoryRepoImpl) Upsert(_ context.Context, history *entity.D
 			First(old).Error
 		switch {
 		case err == nil:
-			return db.Model(old).Select("create_time", "file_size", "first_event_time").Updates(history).Error
+			return db.Model(old).Select("create_time", "file_size", "first_event_time", "last_event_time").Updates(history).Error
 		case errors.Is(err, gorm.ErrRecordNotFound):
 			return db.Create(history).Error
 		default:
@@ -103,9 +104,10 @@ func (repo *dbBinlogHistoryRepoImpl) InsertWithBinlogFiles(ctx context.Context,
 		history := &entity.DbBinlogHistory{
 			CreateTime:     time.Now(),
 			FileName:       fileOnServer.Name,
-			FileSize:       fileOnServer.Size,
+			FileSize:       fileOnServer.RemoteSize,
 			Sequence:       fileOnServer.Sequence,
 			FirstEventTime: fileOnServer.FirstEventTime,
+			LastEventTime:  fileOnServer.LastEventTime,
 			DbInstanceId:   instanceId,
 		}
 		histories = append(histories, history)
@@ -122,3 +124,13 @@ func (repo *dbBinlogHistoryRepoImpl) InsertWithBinlogFiles(ctx context.Context,
 	}
 	return nil
 }
+
+func (repo *dbBinlogHistoryRepoImpl) GetHistoriesBeforeSequence(ctx context.Context, instanceId uint64, binlogSeq int64, histories *[]*entity.DbBinlogHistory) error {
+	return global.Db.Model(repo.GetModel()).
+		Where("db_instance_id = ?", instanceId).
+		Where("sequence < ?", binlogSeq).
+		Scopes(gormx.UndeleteScope).
+		Order("id").
+		Find(histories).
+		Error
+}

server/internal/db/infrastructure/persistence/db_job_base.go

@@ -12,20 +12,12 @@ import (
 	"reflect"
 )
 
-var _ repository.DbJobBase = (*dbJobBaseImpl[entity.DbJob])(nil)
+var _ repository.DbJobBase[entity.DbJob] = (*dbJobBaseImpl[entity.DbJob])(nil)
 
 type dbJobBaseImpl[T entity.DbJob] struct {
 	base.RepoImpl[T]
 }
 
-func (d *dbJobBaseImpl[T]) GetById(e entity.DbJob, id uint64, cols ...string) error {
-	return d.RepoImpl.GetById(e.(T), id, cols...)
-}
-
-func (d *dbJobBaseImpl[T]) UpdateById(ctx context.Context, e entity.DbJob, columns ...string) error {
-	return d.RepoImpl.UpdateById(ctx, e.(T), columns...)
-}
-
 func (d *dbJobBaseImpl[T]) UpdateLastStatus(ctx context.Context, job entity.DbJob) error {
 	return d.UpdateById(ctx, job.(T), "last_status", "last_result", "last_time")
 }

server/internal/db/infrastructure/persistence/db_restore.go

@@ -84,9 +84,9 @@ func (d *dbRestoreRepoImpl) UpdateEnabled(_ context.Context, jobId uint64, enabl
 	cond := map[string]any{
 		"id": jobId,
 	}
-	desc := "任务已禁用"
+	desc := "已禁用"
 	if enabled {
-		desc = "任务已启用"
+		desc = "已启用"
 	}
 	return d.Updates(cond, map[string]any{
 		"enabled":      enabled,

server/internal/db/infrastructure/persistence/instance.go

@@ -12,7 +12,7 @@ type instanceRepoImpl struct {
 	base.RepoImpl[*entity.DbInstance]
 }
 
-func newInstanceRepo() repository.Instance {
+func NewInstanceRepo() repository.Instance {
 	return &instanceRepoImpl{base.RepoImpl[*entity.DbInstance]{M: new(entity.DbInstance)}}
 }
 

server/internal/db/infrastructure/persistence/persistence.go

@@ -5,7 +5,7 @@ import (
 )
 
 func Init() {
-	ioc.Register(newInstanceRepo(), ioc.WithComponentName("DbInstanceRepo"))
+	ioc.Register(NewInstanceRepo(), ioc.WithComponentName("DbInstanceRepo"))
 	ioc.Register(newDbRepo(), ioc.WithComponentName("DbRepo"))
 	ioc.Register(newDbSqlRepo(), ioc.WithComponentName("DbSqlRepo"))
 	ioc.Register(newDbSqlExecRepo(), ioc.WithComponentName("DbSqlExecRepo"))