TRKJ/mayfly-go
2
0
Fork 0
mirror of https://gitee.com/dromara/mayfly-go synced 2025-11-04 00:10:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 数据库迁移至文件支持文件保存天数等

Browse Source
This commit is contained in:
meilin.huang
2024-10-22 20:39:44 +08:00
parent ea3c70a8a8
commit 44a1bd626e
19 changed files with 1043 additions and 1041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/internal/db/api/form/db_transfer.go
View File

@@ -8,6 +8,7 @@ type DbTransferTaskForm struct {
Cron string `json:"cron"` // 定时任务cron表达式
Mode int `binding:"required" json:"mode"` // 数据迁移方式，1、迁移到数据库 2、迁移到文件
TargetFileDbType string `json:"targetFileDbType"` // 目标文件数据库类型
FileSaveDays int `json:"fileSaveDays"` // 文件保存天数
Status int `json:"status" form:"status"` // 启用状态 1启用 -1禁用
CheckedKeys string `binding:"required" json:"checkedKeys"` // 选中需要迁移的表

1
server/internal/db/api/vo/db_transfer.go
View File

@@ -19,6 +19,7 @@ type DbTransferTaskListVO struct {
Cron string `json:"cron"` // 定时任务cron表达式
Mode int `json:"mode"` // 数据迁移方式，1、迁移到数据库 2、迁移到文件
TargetFileDbType string `json:"targetFileDbType"` // 目标文件数据库类型
FileSaveDays int `json:"fileSaveDays"` // 文件保存天数
CheckedKeys string `json:"checkedKeys"` // 选中需要迁移的表
DeleteTable int `json:"deleteTable"` // 创建表前是否删除表

3
server/internal/db/application/application.go
View File

@@ -31,8 +31,10 @@ func Init() {
//if err := GetDbBinlogApp().Init(); err != nil {
// panic(fmt.Sprintf("初始化 DbBinlogApp 失败: %v", err))
//}
GetDataSyncTaskApp().InitCronJob()
GetDbTransferTaskApp().InitCronJob()
GetDbTransferTaskApp().TimerDeleteTransferFile()
InitDbFlowHandler()
})()
}
@@ -56,6 +58,7 @@ func GetDbBinlogApp() *DbBinlogApp {
func GetDataSyncTaskApp() DataSyncTask {
return ioc.Get[DataSyncTask]("DbDataSyncTaskApp")
}
func GetDbTransferTaskApp() DbTransferTask {
return ioc.Get[DbTransferTask]("DbTransferTaskApp")
}

29
server/internal/db/application/db_transfer.go
View File

@@ -53,6 +53,9 @@ type DbTransferTask interface {
IsRunning(taskId uint64) bool
Stop(ctx context.Context, taskId uint64) error
// TimerDeleteTransferFile 定时删除迁移文件
TimerDeleteTransferFile()
}
type dbTransferAppImpl struct {
@@ -80,6 +83,9 @@ func (app *dbTransferAppImpl) Save(ctx context.Context, taskEntity *entity.DbTra
} else {
err = app.UpdateById(ctx, taskEntity)
}
if err != nil {
return err
}
// 视情况添加或删除任务
task, err := app.GetById(taskEntity.Id)
if err != nil {
@@ -549,6 +555,29 @@ func (app *dbTransferAppImpl) transferIndex(_ context.Context, tableInfo dbi.Tab
return targetDialect.CreateIndex(tableInfo, indexs)
}
func (d *dbTransferAppImpl) TimerDeleteTransferFile() {
logx.Debug("开始定时删除迁移文件...")
scheduler.AddFun("@every 100m", func() {
dts, err := d.ListByCond(model.NewCond().Eq("mode", entity.DbTransferTaskModeFile).Ge("file_save_days", 1))
if err != nil {
logx.Errorf("定时获取数据库迁移至文件任务失败: %s", err.Error())
return
}
for _, dt := range dts {
needDelFiles, err := d.transferFileApp.ListByCond(model.NewCond().Eq("task_id", dt.Id).Le("create_time", time.Now().AddDate(0, 0, -dt.FileSaveDays)))
if err != nil {
logx.Errorf("定时获取迁移文件失败: %s", err.Error())
continue
}
for _, nf := range needDelFiles {
if err := d.transferFileApp.Delete(context.Background(), nf.Id); err != nil {
logx.Errorf("定时删除迁移文件失败: %s", err.Error())
}
}
}
})
}
// MarkRunning 标记任务执行中
func (app *dbTransferAppImpl) MarkRunning(taskId uint64) {
cache.Set(fmt.Sprintf("mayfly:db:transfer:%d", taskId), 1, -1)

2
server/internal/db/application/db_transfer_file.go
View File

@@ -54,7 +54,7 @@ func (app *dbTransferFileAppImpl) Delete(ctx context.Context, id ...uint64) erro
// 删除对应的文件
for _, file := range arr {
_ = app.fileApp.Remove(ctx, file.FileKey)
app.fileApp.Remove(ctx, file.FileKey)
}
// 删除数据

4
server/internal/db/dbm/sqlite/metadata.go
View File

@@ -89,7 +89,7 @@ func (sd *SqliteMetaData) GetTables(tableNames ...string) ([]dbi.Table, error) {
func (sd *SqliteMetaData) getDataTypes(dataType string) (string, string, string) {
matches := dataTypeRegexp.FindStringSubmatch(dataType)
if len(matches) == 0 {
return "", "", ""
return dataType, "", ""
}
return matches[1], matches[2], matches[3]
}
@@ -133,7 +133,7 @@ func (sd *SqliteMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
} else {
column.CharMaxLength = cast.ToInt(length)
}
column.DataType = dbi.ColumnDataType(dataType)
column.DataType = dbi.ColumnDataType(strings.ToLower(dataType))
columnHelper.FixColumn(&column)
columns = append(columns, column)

1
server/internal/db/domain/entity/db_transfer.go
View File

@@ -15,6 +15,7 @@ type DbTransferTask struct {
Cron string `orm:"column(cron)" json:"cron"` // 定时任务cron表达式
Mode int8 `orm:"column(mode)" json:"mode"` // 数据迁移方式，1、迁移到数据库 2、迁移到文件
TargetFileDbType string `orm:"column(target_file_db_type)" json:"targetFileDbType"` // 目标文件数据库类型
FileSaveDays int `json:"fileSaveDays"` // 文件保存天数
TaskKey string `orm:"column(key)" json:"taskKey"` // 定时任务唯一uuid key
CheckedKeys string `orm:"column(checked_keys)" json:"checkedKeys"` // 选中需要迁移的表

26
server/internal/machine/application/application.go
View File

@@ -1,7 +1,13 @@
package application
import (
"context"
"mayfly-go/internal/event"
"mayfly-go/internal/machine/domain/entity"
"mayfly-go/pkg/eventbus"
"mayfly-go/pkg/global"
"mayfly-go/pkg/ioc"
"sync"
)
func InitIoc() {
@@ -13,6 +19,26 @@ func InitIoc() {
ioc.Register(new(machineCmdConfAppImpl), ioc.WithComponentName("MachineCmdConfApp"))
}
func Init() {
sync.OnceFunc(func() {
GetMachineCronJobApp().InitCronJob()
GetMachineApp().TimerUpdateStats()
GetMachineTermOpApp().TimerDeleteTermOp()
global.EventBus.Subscribe(event.EventTopicDeleteMachine, "machineFile", func(ctx context.Context, event *eventbus.Event) error {
me := event.Val.(*entity.Machine)
return GetMachineFileApp().DeleteByCond(ctx, &entity.MachineFile{MachineId: me.Id})
})
global.EventBus.Subscribe(event.EventTopicDeleteMachine, "machineScript", func(ctx context.Context, event *eventbus.Event) error {
me := event.Val.(*entity.Machine)
return GetMachineScriptApp().DeleteByCond(ctx, &entity.MachineScript{MachineId: me.Id})
})
})()
}
func GetMachineApp() Machine {
return ioc.Get[Machine]("MachineApp")
}

25
server/internal/machine/init/init.go
View File

@@ -1,15 +1,10 @@
package init
import (
"context"
"mayfly-go/initialize"
"mayfly-go/internal/event"
"mayfly-go/internal/machine/application"
"mayfly-go/internal/machine/domain/entity"
"mayfly-go/internal/machine/infrastructure/persistence"
"mayfly-go/internal/machine/router"
"mayfly-go/pkg/eventbus"
"mayfly-go/pkg/global"
)
func init() {
@@ -18,23 +13,5 @@ func init() {
application.InitIoc()
})
initialize.AddInitRouterFunc(router.Init)
initialize.AddInitFunc(Init)
}
func Init() {
application.GetMachineCronJobApp().InitCronJob()
application.GetMachineApp().TimerUpdateStats()
application.GetMachineTermOpApp().TimerDeleteTermOp()
global.EventBus.Subscribe(event.EventTopicDeleteMachine, "machineFile", func(ctx context.Context, event *eventbus.Event) error {
me := event.Val.(*entity.Machine)
return application.GetMachineFileApp().DeleteByCond(ctx, &entity.MachineFile{MachineId: me.Id})
})
global.EventBus.Subscribe(event.EventTopicDeleteMachine, "machineScript", func(ctx context.Context, event *eventbus.Event) error {
me := event.Val.(*entity.Machine)
return application.GetMachineScriptApp().DeleteByCond(ctx, &entity.MachineScript{MachineId: me.Id})
})
initialize.AddInitFunc(application.Init)
}

2
server/pkg/config/app.go
View File

@@ -4,7 +4,7 @@ import "fmt"
const (
AppName = "mayfly-go"
Version = "v1.8.9"
Version = "v1.9.0"
)
func GetAppInfo() string {

BIN
server/resources/data/mayfly-go.sqlite
View File

Binary file not shown.

1
server/resources/script/sql/mayfly-go.sql
View File

@@ -69,6 +69,7 @@ CREATE TABLE `t_db_transfer_task` (
`task_key` varchar(100) NULL comment '定时任务唯一uuid key',
`mode` TINYINT(3) NOT NULL DEFAULT 1 comment '数据迁移方式，1、迁移到数据库 2、迁移到文件',
`target_file_db_type` varchar(200) NULL comment '目标文件语言类型，类型枚举同target_db_type',
`file_save_days` int NULL comment '文件保存天数',
`status` tinyint(3) NOT NULL DEFAULT '1' comment '启用状态 1启用 -1禁用',
`upd_field_src` varchar(100) DEFAULT NULL COMMENT '更新值来源字段，默认同更新字段，如果查询结果指定了字段别名且与原更新字段不一致，则取这个字段值为当前更新值',
`delete_time` datetime DEFAULT NULL COMMENT '删除时间',

3
server/resources/script/sql/v1.8/v1.8.9.sql → server/resources/script/sql/v1.8/v1.9.0.sql
View File

@@ -11,7 +11,7 @@ INSERT INTO `t_sys_resource` (`id`, `pid`, `type`, `status`, `name`, `code`, `we
-- 新增数据库迁移相关的系统配置
DELETE FROM `t_sys_config` WHERE `key` = 'DbBackupRestore';
UPDATE `t_sys_config` SET param = '[{"name":"uploadMaxFileSize","model":"uploadMaxFileSize","placeholder":"允许上传的最大文件大小(1MB、2GB等)"},{"model":"termOpSaveDays","name":"终端记录保存时间","placeholder":"终端记录保存时间（单位天）"},{"model":"guacdHost","name":"guacd服务ip","placeholder":"guacd服务ip，默认 127.0.0.1","required":false},{"name":"guacd服务端口","model":"guacdPort","placeholder":"guacd服务端口，默认 4822","required":false},{"model":"guacdFilePath","name":"guacd服务文件存储位置","placeholder":"guacd服务文件存储位置，用于挂载RDP文件夹"}]' WHERE `key`='MachineConfig';
UPDATE `t_sys_config` SET params = '[{"name":"uploadMaxFileSize","model":"uploadMaxFileSize","placeholder":"允许上传的最大文件大小(1MB、2GB等)"},{"model":"termOpSaveDays","name":"终端记录保存时间","placeholder":"终端记录保存时间（单位天）"},{"model":"guacdHost","name":"guacd服务ip","placeholder":"guacd服务ip，默认 127.0.0.1","required":false},{"name":"guacd服务端口","model":"guacdPort","placeholder":"guacd服务端口，默认 4822","required":false},{"model":"guacdFilePath","name":"guacd服务文件存储位置","placeholder":"guacd服务文件存储位置，用于挂载RDP文件夹"}]' WHERE `key`='MachineConfig';
INSERT INTO `t_sys_config` (`name`, `key`, `params`, `value`, `remark`, `permission`, `create_time`, `creator_id`, `creator`, `update_time`, `modifier_id`, `modifier`, `is_deleted`, `delete_time`) VALUES('文件配置', 'FileConfig', '[{"model":"basePath","name":"基础路径","placeholder":"默认为可执行文件对应目录下./file"}]', '{"basePath":"./file"}', '系统文件相关配置', 'admin,', '2024-10-20 22:30:01', 1, 'admin', '2024-10-21 13:51:17', 1, 'admin', 0, NULL);
-- 数据库迁移到文件
@@ -22,6 +22,7 @@ ALTER TABLE `t_db_transfer_task`
ADD COLUMN `task_key` varchar(100) NULL comment '定时任务唯一uuid key',
ADD COLUMN `mode` TINYINT(3) NOT NULL DEFAULT 1 comment '数据迁移方式，1、迁移到数据库 2、迁移到文件',
ADD COLUMN `target_file_db_type` varchar(200) NULL comment '目标文件语言类型，类型枚举同target_db_type',
ADD COLUMN `file_save_days` int NULL comment '文件保存天数',
ADD COLUMN `status` tinyint(3) NOT NULL DEFAULT '1' comment '启用状态 1启用 -1禁用';
UPDATE `t_db_transfer_task` SET mode = 1 WHERE 1=1;