refactor: dbms优化

mayfly_go_web/src/components/terminal/TerminalLog.vue

@@ -9,13 +9,17 @@
                 </DrawerHeader>
             </template>
 
-            <TerminalBody ref="terminalRef" />
+            <el-descriptions class="mb10" :column="1" border v-if="extra">
+                <el-descriptions-item v-for="(value, key) in extra" :key="key" :span="1" :label="key">{{ value }}</el-descriptions-item>
+            </el-descriptions>
+
+            <TerminalBody class="mb10" ref="terminalRef" height="calc(100vh - 220px)" />
         </el-drawer>
     </div>
 </template>
 
 <script lang="ts" setup>
-import { ref, watch } from 'vue';
+import { computed, ref, watch } from 'vue';
 import DrawerHeader from '@/components/drawer-header/DrawerHeader.vue';
 import TerminalBody from './TerminalBody.vue';
 import { logApi } from '../../views/system/api';
@@ -37,6 +41,13 @@ const terminalRef: any = ref(null);
 const nowLine = ref(0);
 const log = ref({}) as any;
 
+const extra = computed(() => {
+    if (log.value?.extra) {
+        return JSON.parse(log.value.extra);
+    }
+    return null;
+});
+
 // 定时获取最新日志
 const { pause, resume } = useIntervalFn(() => {
     writeLog();
@@ -82,9 +93,9 @@ const writeLog2Term = (log: any) => {
     const lines = log.resp.split('\n');
     for (let line of lines.slice(nowLine.value)) {
         nowLine.value += 1;
-        terminalRef.value.writeln2Term(line);
+        terminalRef.value?.writeln2Term(line);
     }
-    terminalRef.value.focus();
+    terminalRef.value?.focus();
 };
 
 const getLog = async () => {

mayfly_go_web/src/views/ops/db/DbList.vue

@@ -62,7 +62,7 @@
                     <template #dropdown>
                         <el-dropdown-menu>
                             <el-dropdown-item :command="{ type: 'detail', data }"> 详情 </el-dropdown-item>
-                            <el-dropdown-item :command="{ type: 'dumpDb', data }" v-if="supportAction('dumpDb', data.type)"> 导出 </el-dropdown-item>
+                            <el-dropdown-item :command="{ type: 'dumpDb', data }"> 导出 </el-dropdown-item>
                             <el-dropdown-item :command="{ type: 'backupDb', data }" v-if="actionBtns[perms.backupDb] && supportAction('backupDb', data.type)">
                                 备份任务
                             </el-dropdown-item>

mayfly_go_web/src/views/ops/db/DbTransferList.vue

@@ -73,8 +73,8 @@ const perms = {
 const searchItems = [SearchItem.input('name', '名称')];
 
 const columns = ref([
-    TableColumn.new('srcDb', '源库').setMinWidth(250).isSlot(),
-    TableColumn.new('targetDb', '目标库').setMinWidth(250).isSlot(),
+    TableColumn.new('srcDb', '源库').setMinWidth(200).isSlot(),
+    TableColumn.new('targetDb', '目标库').setMinWidth(200).isSlot(),
     TableColumn.new('runningState', '执行状态').typeTag(DbTransferRunningStateEnum),
     TableColumn.new('creator', '创建人'),
     TableColumn.new('createTime', '创建时间').isTime(),

mayfly_go_web/src/views/ops/db/component/sqleditor/SqlExecDialog.vue

@@ -2,7 +2,13 @@
     <div>
         <el-dialog title="待执行SQL" v-model="dialogVisible" :show-close="false" width="600px">
             <monaco-editor height="300px" class="codesql" language="sql" v-model="sqlValue" />
-            <el-input @keyup.enter="runSql" ref="remarkInputRef" v-model="remark" placeholder="请输入执行备注" class="mt5" />
+            <el-input
+                @keyup.enter="runSql"
+                ref="remarkInputRef"
+                v-model="remark"
+                :placeholder="props.flowProcdefKey ? '执行备注（必填）' : '执行备注（选填）'"
+                class="mt5"
+            />
 
             <div v-if="props.flowProcdefKey">
                 <el-divider content-position="left">审批节点</el-divider>
@@ -52,7 +58,8 @@ onMounted(() => {
  * 执行sql
  */
 const runSql = async () => {
-    if (!state.remark) {
+    // 存在流程审批，则备注为必填
+    if (!state.remark && props.flowProcdefKey) {
         ElMessage.error('请输入执行的备注信息');
         return;
     }

mayfly_go_web/src/views/ops/db/component/table/DbTableDataOp.vue

@@ -205,8 +205,8 @@
             </el-col>
         </el-row>
 
-        <el-dialog v-model="conditionDialog.visible" :title="conditionDialog.title" width="420px">
-            <el-row>
+        <el-dialog v-model="conditionDialog.visible" :title="conditionDialog.title" width="460px">
+            <el-row gutter="5">
                 <el-col :span="5">
                     <el-select v-model="conditionDialog.condition">
                         <el-option label="=" value="="> </el-option>

mayfly_go_web/src/views/system/syslog/SyslogList.vue

@@ -40,8 +40,8 @@ const searchItems = [
 const columns = [
     TableColumn.new('creator', '操作人').isSlot().noShowOverflowTooltip(),
     TableColumn.new('createTime', '操作时间').isTime(),
-    TableColumn.new('type', '结果').typeTag(LogTypeEnum),
     TableColumn.new('description', '描述'),
+    TableColumn.new('type', '结果').typeTag(LogTypeEnum),
     TableColumn.new('reqParam', '操作信息').canBeautify(),
     TableColumn.new('resp', '响应信息').canBeautify(),
 ];

server/internal/db/api/db_data_sync.go

@@ -66,7 +66,7 @@ func (d *DataSyncTask) ChangeStatus(rc *req.Ctx) {
 	if task.Status == entity.DataSyncTaskStatusEnable {
 		task, err := d.DataSyncTaskApp.GetById(new(entity.DataSyncTask), task.Id)
 		biz.ErrIsNil(err, "该任务不存在")
-		d.DataSyncTaskApp.AddCronJob(task)
+		d.DataSyncTaskApp.AddCronJob(rc.MetaCtx, task)
 	} else {
 		d.DataSyncTaskApp.RemoveCronJobById(task.Id)
 	}
@@ -77,7 +77,7 @@ func (d *DataSyncTask) ChangeStatus(rc *req.Ctx) {
 func (d *DataSyncTask) Run(rc *req.Ctx) {
 	taskId := d.getTaskId(rc)
 	rc.ReqParam = taskId
-	_ = d.DataSyncTaskApp.RunCronJob(taskId)
+	_ = d.DataSyncTaskApp.RunCronJob(rc.MetaCtx, taskId)
 }
 
 func (d *DataSyncTask) Stop(rc *req.Ctx) {

server/internal/db/application/db.go

@@ -222,6 +222,9 @@ func (d *dbAppImpl) DumpDb(ctx context.Context, reqParam *DumpDbReq) error {
 			tables[i] = table.TableName
 		}
 	}
+	if len(tables) == 0 {
+		return errorx.NewBiz("不存在可导出的表")
+	}
 
 	// 查询列信息，后面生成建表ddl和insert都需要列信息
 	columns, err := dbMeta.GetColumns(tables...)

server/internal/db/application/db_data_sync.go

@@ -9,6 +9,7 @@ import (
 	"mayfly-go/internal/db/domain/entity"
 	"mayfly-go/internal/db/domain/repository"
 	"mayfly-go/pkg/base"
+	"mayfly-go/pkg/contextx"
 	"mayfly-go/pkg/errorx"
 	"mayfly-go/pkg/gormx"
 	"mayfly-go/pkg/logx"
@@ -34,11 +35,11 @@ type DataSyncTask interface {
 
 	InitCronJob()
 
-	AddCronJob(taskEntity *entity.DataSyncTask)
+	AddCronJob(ctx context.Context, taskEntity *entity.DataSyncTask)
 
 	RemoveCronJobById(taskId uint64)
 
-	RunCronJob(id uint64) error
+	RunCronJob(ctx context.Context, id uint64) error
 
 	GetTaskLogList(condition *entity.DataSyncLogQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
 }
@@ -82,7 +83,7 @@ func (app *dataSyncAppImpl) Save(ctx context.Context, taskEntity *entity.DataSyn
 	if err != nil {
 		return err
 	}
-	app.AddCronJob(task)
+	app.AddCronJob(ctx, task)
 	return nil
 }
 
@@ -94,7 +95,7 @@ func (app *dataSyncAppImpl) Delete(ctx context.Context, id uint64) error {
 	return nil
 }
 
-func (app *dataSyncAppImpl) AddCronJob(taskEntity *entity.DataSyncTask) {
+func (app *dataSyncAppImpl) AddCronJob(ctx context.Context, taskEntity *entity.DataSyncTask) {
 	key := taskEntity.TaskKey
 	// 先移除旧的任务
 	scheduler.RemoveByKey(key)
@@ -104,7 +105,7 @@ func (app *dataSyncAppImpl) AddCronJob(taskEntity *entity.DataSyncTask) {
 		taskId := taskEntity.Id
 		scheduler.AddFunByKey(key, taskEntity.TaskCron, func() {
 			logx.Infof("开始执行同步任务: %d", taskId)
-			if err := app.RunCronJob(taskId); err != nil {
+			if err := app.RunCronJob(ctx, taskId); err != nil {
 				logx.Errorf("定时执行数据同步任务失败: %s", err.Error())
 			}
 		})
@@ -125,7 +126,7 @@ func (app *dataSyncAppImpl) changeRunningState(id uint64, state int8) {
 	_ = app.UpdateById(context.Background(), task)
 }
 
-func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
+func (app *dataSyncAppImpl) RunCronJob(ctx context.Context, id uint64) error {
 	// 查询最新的任务信息
 	task, err := app.GetById(new(entity.DataSyncTask), id)
 	if err != nil {
@@ -137,7 +138,7 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
 	// 开始运行时，修改状态为运行中
 	app.changeRunningState(id, entity.DataSyncTaskRunStateRunning)
 
-	logx.Infof("开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
+	logx.InfofContext(ctx, "开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
 
 	go func() {
 		// 通过占位符格式化sql
@@ -146,7 +147,7 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
 		if task.UpdFieldVal != "0" && task.UpdFieldVal != "" && task.UpdField != "" {
 			srcConn, err := app.dbApp.GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
 			if err != nil {
-				logx.Errorf("数据源连接不可用, %s", err.Error())
+				logx.ErrorfContext(ctx, "数据源连接不可用, %s", err.Error())
 				return
 			}
 
@@ -177,9 +178,10 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
 		// 组装查询sql
 		sql := fmt.Sprintf("%s %s %s %s", task.DataSql, where, updSql, orderSql)
 
-		log, err := app.doDataSync(sql, task)
+		log, err := app.doDataSync(ctx, sql, task)
 		if err != nil {
 			log.ErrText = fmt.Sprintf("执行失败: %s", err.Error())
+			logx.ErrorContext(ctx, log.ErrText)
 			log.Status = entity.DataSyncTaskStateFail
 			app.endRunning(task, log)
 		}
@@ -188,7 +190,7 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
 	return nil
 }
 
-func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*entity.DataSyncLog, error) {
+func (app *dataSyncAppImpl) doDataSync(ctx context.Context, sql string, task *entity.DataSyncTask) (*entity.DataSyncLog, error) {
 	now := time.Now()
 	syncLog := &entity.DataSyncLog{
 		TaskId:      task.Id,
@@ -264,6 +266,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
 
 			// 记录当前已同步的数据量
 			syncLog.ErrText = fmt.Sprintf("本次任务执行中，已同步：%d条", total)
+			logx.InfoContext(ctx, syncLog.ErrText)
 			syncLog.ResNum = total
 			app.saveLog(syncLog)
 
@@ -293,7 +296,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
 		}
 	}
 
-	logx.Infof("同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
+	logx.InfofContext(ctx, "同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
 
 	// 保存执行成功日志
 	syncLog.ErrText = fmt.Sprintf("本次任务执行成功，新数据：%d 条", total)
@@ -430,7 +433,7 @@ func (app *dataSyncAppImpl) InitCronJob() {
 
 	for {
 		for _, job := range *jobs {
-			app.AddCronJob(&job)
+			app.AddCronJob(contextx.NewTraceId(), &job)
 			add++
 		}
 		if add >= int(total) {

server/internal/db/application/db_transfer.go

@@ -9,6 +9,7 @@ import (
 	sysapp "mayfly-go/internal/sys/application"
 	sysentity "mayfly-go/internal/sys/domain/entity"
 	"mayfly-go/pkg/base"
+	"mayfly-go/pkg/cache"
 	"mayfly-go/pkg/errorx"
 	"mayfly-go/pkg/gormx"
 	"mayfly-go/pkg/logx"
@@ -17,6 +18,8 @@ import (
 	"sort"
 	"strings"
 	"time"
+
+	"golang.org/x/sync/errgroup"
 )
 
 type DbTransferTask interface {
@@ -95,12 +98,19 @@ func (app *dbTransferAppImpl) Run(ctx context.Context, taskId uint64) {
 		logx.Errorf("创建DBMS-执行数据迁移日志失败：%v", err)
 		return
 	}
-	defer app.logApp.Flush(logId)
+	defer app.logApp.Flush(logId, true)
 
 	// 修改状态与关联日志id
 	task.LogId = logId
 	task.RunningState = entity.DbTransferTaskRunStateRunning
-	app.UpdateById(ctx, task)
+	if err = app.UpdateById(ctx, task); err != nil {
+		logx.Errorf("更新任务执行状态失败")
+		return
+	}
+
+	// 标记该任务开始执行
+	app.MarkRuning(taskId)
+	defer app.MarkStop(taskId)
 
 	// 获取源库连接、目标库连接，判断连接可用性，否则记录日志：xx连接不可用
 	// 获取源库表信息
@@ -142,40 +152,6 @@ func (app *dbTransferAppImpl) Run(ctx context.Context, taskId uint64) {
 	app.EndTransfer(ctx, logId, taskId, fmt.Sprintf("执行迁移完成，执行迁移任务[taskId = %d]完成, 耗时：%v", taskId, time.Since(start)), nil, nil)
 }
 
-func (app *dbTransferAppImpl) Log(ctx context.Context, logId uint64, msg string) {
-	logType := sysentity.SyslogTypeRunning
-	logx.InfoContext(ctx, msg)
-	app.logApp.AppendLog(logId, &sysapp.AppendLogReq{
-		AppendResp: msg,
-		Type:       logType,
-	})
-}
-
-func (app *dbTransferAppImpl) EndTransfer(ctx context.Context, logId uint64, taskId uint64, msg string, err error, extra map[string]any) {
-	logType := sysentity.SyslogTypeSuccess
-	transferState := entity.DbTransferTaskRunStateSuccess
-	if err != nil {
-		msg = fmt.Sprintf("%s: %s", msg, err.Error())
-		logx.ErrorContext(ctx, msg)
-		logType = sysentity.SyslogTypeError
-		transferState = entity.DbTransferTaskRunStateFail
-	} else {
-		logx.InfoContext(ctx, msg)
-	}
-
-	app.logApp.AppendLog(logId, &sysapp.AppendLogReq{
-		AppendResp: msg,
-		Extra:      extra,
-		Type:       logType,
-	})
-
-	// 修改任务状态
-	task := new(entity.DbTransferTask)
-	task.Id = taskId
-	task.RunningState = transferState
-	app.UpdateById(context.Background(), task)
-}
-
 func (app *dbTransferAppImpl) Stop(ctx context.Context, taskId uint64) error {
 	task, err := app.GetById(new(entity.DbTransferTask), taskId)
 	if err != nil {
@@ -186,7 +162,12 @@ func (app *dbTransferAppImpl) Stop(ctx context.Context, taskId uint64) error {
 		return errorx.NewBiz("该任务未在执行")
 	}
 	task.RunningState = entity.DbTransferTaskRunStateStop
-	return app.UpdateById(ctx, task)
+	if err = app.UpdateById(ctx, task); err != nil {
+		return err
+	}
+
+	app.MarkStop(taskId)
+	return nil
 }
 
 // 迁移表
@@ -222,50 +203,63 @@ func (app *dbTransferAppImpl) transferTables(ctx context.Context, logId uint64,
 	srcColumnHelper := srcMeta.GetColumnHelper()
 	targetColumnHelper := targetConn.GetMetaData().GetColumnHelper()
 
-	for _, tbName := range sortTableNames {
-		cols := columnMap[tbName]
-		targetCols := make([]dbi.Column, 0)
-		for _, col := range cols {
-			colPtr := &col
-			// 源库列转为公共列
-			srcColumnHelper.ToCommonColumn(colPtr)
-			// 公共列转为目标库列
-			targetColumnHelper.ToColumn(colPtr)
-			targetCols = append(targetCols, *colPtr)
-		}
+	// 分组迁移
+	tableGroups := collx.ArraySplit[string](sortTableNames, 2)
+	errGroup, _ := errgroup.WithContext(ctx)
 
-		// 通过公共列信息生成目标库的建表语句，并执行目标库建表
-		app.Log(ctx, logId, fmt.Sprintf("开始创建目标表: 表名：%s", tbName))
-		_, err := targetDialect.CreateTable(targetCols, tableMap[tbName], true)
-		if err != nil {
-			return errorx.NewBiz(fmt.Sprintf("创建目标表失败: 表名：%s, error: %s", tbName, err.Error()))
-		}
-		app.Log(ctx, logId, fmt.Sprintf("创建目标表成功: 表名：%s", tbName))
+	for _, tables := range tableGroups {
+		errGroup.Go(func() error {
+			for _, tbName := range tables {
+				cols := columnMap[tbName]
+				targetCols := make([]dbi.Column, 0)
+				for _, col := range cols {
+					colPtr := &col
+					// 源库列转为公共列
+					srcColumnHelper.ToCommonColumn(colPtr)
+					// 公共列转为目标库列
+					targetColumnHelper.ToColumn(colPtr)
+					targetCols = append(targetCols, *colPtr)
+				}
 
-		// 迁移数据
-		app.Log(ctx, logId, fmt.Sprintf("开始迁移数据: 表名：%s", tbName))
-		total, err := app.transferData(ctx, task.Id, tbName, targetCols, srcConn, targetConn)
-		if err != nil {
-			return errorx.NewBiz(fmt.Sprintf("迁移数据失败: 表名：%s, error: %s", tbName, err.Error()))
-		}
-		app.Log(ctx, logId, fmt.Sprintf("迁移数据成功: 表名：%s, 数据：%d 条", tbName, total))
+				// 通过公共列信息生成目标库的建表语句，并执行目标库建表
+				app.Log(ctx, logId, fmt.Sprintf("开始创建目标表: 表名：%s", tbName))
+				_, err := targetDialect.CreateTable(targetCols, tableMap[tbName], true)
+				if err != nil {
+					return errorx.NewBiz(fmt.Sprintf("创建目标表失败: 表名：%s, error: %s", tbName, err.Error()))
+				}
+				app.Log(ctx, logId, fmt.Sprintf("创建目标表成功: 表名：%s", tbName))
 
-		// 有些数据库迁移完数据之后，需要更新表自增序列为当前表最大值
-		targetDialect.UpdateSequence(tbName, targetCols)
+				// 迁移数据
+				app.Log(ctx, logId, fmt.Sprintf("开始迁移数据: 表名：%s", tbName))
+				total, err := app.transferData(ctx, logId, task.Id, tbName, targetCols, srcConn, targetConn)
+				if err != nil {
+					return errorx.NewBiz(fmt.Sprintf("迁移数据失败: 表名：%s, error: %s", tbName, err.Error()))
+				}
+				app.Log(ctx, logId, fmt.Sprintf("迁移数据成功: 表名：%s, 数据：%d 条", tbName, total))
 
-		// 迁移索引信息
-		app.Log(ctx, logId, fmt.Sprintf("开始迁移索引: 表名：%s", tbName))
-		err = app.transferIndex(ctx, tableMap[tbName], srcConn, targetDialect)
-		if err != nil {
-			return errorx.NewBiz(fmt.Sprintf("迁移索引失败: 表名：%s, error: %s", tbName, err.Error()))
-		}
-		app.Log(ctx, logId, fmt.Sprintf("迁移索引成功: 表名：%s", tbName))
+				// 有些数据库迁移完数据之后，需要更新表自增序列为当前表最大值
+				targetDialect.UpdateSequence(tbName, targetCols)
+
+				// 迁移索引信息
+				app.Log(ctx, logId, fmt.Sprintf("开始迁移索引: 表名：%s", tbName))
+				err = app.transferIndex(ctx, tableMap[tbName], srcConn, targetDialect)
+				if err != nil {
+					return errorx.NewBiz(fmt.Sprintf("迁移索引失败: 表名：%s, error: %s", tbName, err.Error()))
+				}
+				app.Log(ctx, logId, fmt.Sprintf("迁移索引成功: 表名：%s", tbName))
+			}
+
+			return nil
+		})
 	}
 
+	if err := errGroup.Wait(); err != nil {
+		return err
+	}
 	return nil
 }
 
-func (app *dbTransferAppImpl) transferData(ctx context.Context, taskId uint64, tableName string, targetColumns []dbi.Column, srcConn *dbi.DbConn, targetConn *dbi.DbConn) (int, error) {
+func (app *dbTransferAppImpl) transferData(ctx context.Context, logId uint64, taskId uint64, tableName string, targetColumns []dbi.Column, srcConn *dbi.DbConn, targetConn *dbi.DbConn) (int, error) {
 	result := make([]map[string]any, 0)
 	total := 0        // 总条数
 	batchSize := 1000 // 每次查询并迁移1000条数据
@@ -273,6 +267,7 @@ func (app *dbTransferAppImpl) transferData(ctx context.Context, taskId uint64, t
 	srcMeta := srcConn.GetMetaData()
 	srcConverter := srcMeta.GetDataHelper()
 	targetDialect := targetConn.GetDialect()
+	logExtraKey := fmt.Sprintf("`%s` 当前已迁移数据量: ", tableName)
 
 	// 游标查询源表数据，并批量插入目标表
 	err = srcConn.WalkTableRows(context.Background(), tableName, func(row map[string]any, columns []*dbi.QueryColumn) error {
@@ -291,6 +286,7 @@ func (app *dbTransferAppImpl) transferData(ctx context.Context, taskId uint64, t
 				return err
 			}
 			result = result[:0]
+			app.logApp.SetExtra(logId, logExtraKey, total)
 		}
 		return nil
 	})
@@ -307,15 +303,13 @@ func (app *dbTransferAppImpl) transferData(ctx context.Context, taskId uint64, t
 			return 0, err
 		}
 	}
+	// 置空当前表数据迁移量进度
+	app.logApp.SetExtra(logId, logExtraKey, nil)
 	return total, err
 }
 
 func (app *dbTransferAppImpl) transfer2Target(taskId uint64, targetConn *dbi.DbConn, targetColumns []dbi.Column, result []map[string]any, targetDialect dbi.Dialect, tbName string) error {
-	task, err := app.GetById(new(entity.DbTransferTask), taskId)
-	if err != nil {
-		return errorx.NewBiz("任务不存在")
-	}
-	if task.RunningState == entity.DbTransferTaskRunStateStop {
+	if !app.IsRunning(taskId) {
 		return errorx.NewBiz("迁移终止")
 	}
 
@@ -376,10 +370,55 @@ func (app *dbTransferAppImpl) transferIndex(_ context.Context, tableInfo dbi.Tab
 		return nil
 	}
 
-	if len(indexs) == 0 {
-		return nil
-	}
-
 	// 通过表名、索引信息生成建索引语句，并执行到目标表
 	return targetDialect.CreateIndex(tableInfo, indexs)
 }
+
+// MarkRuning 标记任务执行中
+func (app *dbTransferAppImpl) MarkRuning(taskId uint64) {
+	cache.Set(fmt.Sprintf("mayfly:db:transfer:%d", taskId), 1, -1)
+}
+
+// MarkStop 标记任务结束
+func (app *dbTransferAppImpl) MarkStop(taskId uint64) {
+	cache.Del(fmt.Sprintf("mayfly:db:transfer:%d", taskId))
+}
+
+// IsRunning 判断任务是否执行中
+func (app *dbTransferAppImpl) IsRunning(taskId uint64) bool {
+	return cache.GetStr(fmt.Sprintf("mayfly:db:transfer:%d", taskId)) != ""
+}
+
+func (app *dbTransferAppImpl) Log(ctx context.Context, logId uint64, msg string, extra ...any) {
+	logType := sysentity.SyslogTypeRunning
+	logx.InfoContext(ctx, msg)
+	app.logApp.AppendLog(logId, &sysapp.AppendLogReq{
+		AppendResp: msg,
+		Type:       logType,
+	})
+}
+
+func (app *dbTransferAppImpl) EndTransfer(ctx context.Context, logId uint64, taskId uint64, msg string, err error, extra map[string]any) {
+	logType := sysentity.SyslogTypeSuccess
+	transferState := entity.DbTransferTaskRunStateSuccess
+	if err != nil {
+		msg = fmt.Sprintf("%s: %s", msg, err.Error())
+		logx.ErrorContext(ctx, msg)
+		logType = sysentity.SyslogTypeError
+		transferState = entity.DbTransferTaskRunStateFail
+	} else {
+		logx.InfoContext(ctx, msg)
+	}
+
+	app.logApp.AppendLog(logId, &sysapp.AppendLogReq{
+		AppendResp: msg,
+		Extra:      extra,
+		Type:       logType,
+	})
+
+	// 修改任务状态
+	task := new(entity.DbTransferTask)
+	task.Id = taskId
+	task.RunningState = transferState
+	app.UpdateById(context.Background(), task)
+}

server/internal/db/dbm/dbi/conn.go

@@ -72,12 +72,12 @@ func (d *DbConn) Query2Struct(execSql string, dest any) error {
 	return scanAll(rows, dest, false)
 }
 
-// 游标方式遍历查询结果集, walkFn返回error不为nil, 则跳出遍历
+// WalkQueryRows 游标方式遍历查询结果集, walkFn返回error不为nil, 则跳出遍历并取消查询
 func (d *DbConn) WalkQueryRows(ctx context.Context, querySql string, walkFn WalkQueryRowsFunc, args ...any) error {
 	return walkQueryRows(ctx, d.db, querySql, walkFn, args...)
 }
 
-// 游标方式遍历指定表的结果集, walkFn返回error不为nil, 则跳出遍历
+// WalkTableRows 游标方式遍历指定表的结果集, walkFn返回error不为nil, 则跳出遍历并取消查询
 func (d *DbConn) WalkTableRows(ctx context.Context, tableName string, walkFn WalkQueryRowsFunc) error {
 	return d.WalkQueryRows(ctx, fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
 }
@@ -117,22 +117,22 @@ func (d *DbConn) TxExecContext(ctx context.Context, tx *sql.Tx, execSql string,
 	return res.RowsAffected()
 }
 
-// 开启事务
+// Begin 开启事务
 func (d *DbConn) Begin() (*sql.Tx, error) {
 	return d.db.Begin()
 }
 
-// 获取数据库dialect实现接口
+// GetDialect 获取数据库dialect实现接口
 func (d *DbConn) GetDialect() Dialect {
 	return d.Info.Meta.GetDialect(d)
 }
 
-// 获取数据库MetaData
+// GetMetaData 获取数据库MetaData
 func (d *DbConn) GetMetaData() *MetaDataX {
 	return d.Info.Meta.GetMetaData(d)
 }
 
-// 返回数据库连接状态
+// Stats 返回数据库连接状态
 func (d *DbConn) Stats(ctx context.Context, execSql string, args ...any) sql.DBStats {
 	return d.db.Stats()
 }
@@ -198,7 +198,7 @@ func walkQueryRows(ctx context.Context, db *sql.DB, selectSql string, walkFn Wal
 			rowData[cols[i].Name] = valueConvert(v, colTypes[i])
 		}
 		if err = walkFn(rowData, cols); err != nil {
-			logx.ErrorfContext(ctx, "游标遍历查询结果集出错, 退出遍历: %s", err.Error())
+			logx.ErrorfContext(ctx, "[%s]游标遍历查询结果集出错, 退出遍历: %s", selectSql, err.Error())
 			cancelFunc()
 			return err
 		}

server/internal/sys/application/syslog.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"mayfly-go/internal/sys/domain/entity"
 	"mayfly-go/internal/sys/domain/repository"
+	"mayfly-go/pkg/cache"
 	"mayfly-go/pkg/contextx"
 	"mayfly-go/pkg/errorx"
 	"mayfly-go/pkg/logx"
@@ -16,7 +17,6 @@ import (
 	"mayfly-go/pkg/utils/jsonx"
 	"mayfly-go/pkg/utils/structx"
 	"mayfly-go/pkg/utils/timex"
-	"sync"
 	"time"
 )
 
@@ -48,15 +48,15 @@ type Syslog interface {
 	// AppendLog 追加日志信息
 	AppendLog(logId uint64, appendLog *AppendLogReq)
 
-	// Flush 实时追加的日志到库里
-	Flush(logId uint64)
+	// SetExtra 设置指定日志的extra信息, val为空则移除该key
+	SetExtra(logId uint64, key string, val any)
+
+	// Flush 实时追加的日志到数据库里
+	Flush(logId uint64, clearExtra bool)
 }
 
 type syslogAppImpl struct {
 	SyslogRepo repository.Syslog `inject:""`
-
-	appendLogs map[uint64]*entity.SysLog
-	rwLock     sync.RWMutex
 }
 
 func (m *syslogAppImpl) GetPageList(condition *entity.SysLogQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
@@ -110,17 +110,14 @@ func (m *syslogAppImpl) SaveFromReq(req *req.Ctx) {
 }
 
 func (m *syslogAppImpl) GetLogDetail(logId uint64) *entity.SysLog {
-	syslog := new(entity.SysLog)
+	syslog := m.GetCacheLog(logId)
+	if syslog != nil {
+		return syslog
+	}
+	syslog = new(entity.SysLog)
 	if err := m.SyslogRepo.GetById(syslog, logId); err != nil {
 		return nil
 	}
-
-	if syslog.Type == entity.SyslogTypeRunning {
-		m.rwLock.RLock()
-		defer m.rwLock.RUnlock()
-		return m.appendLogs[logId]
-	}
-
 	return syslog
 }
 
@@ -128,7 +125,7 @@ func (m *syslogAppImpl) CreateLog(ctx context.Context, log *CreateLogReq) (uint6
 	syslog := new(entity.SysLog)
 	structx.Copy(syslog, log)
 	syslog.ReqParam = anyx.ToString(log.ReqParam)
-	if log.Extra != nil {
+	if len(log.Extra) > 0 {
 		syslog.Extra = jsonx.ToStr(log.Extra)
 	}
 	if err := m.SyslogRepo.Insert(ctx, syslog); err != nil {
@@ -138,34 +135,52 @@ func (m *syslogAppImpl) CreateLog(ctx context.Context, log *CreateLogReq) (uint6
 }
 
 func (m *syslogAppImpl) AppendLog(logId uint64, appendLog *AppendLogReq) {
-	m.rwLock.Lock()
-	defer m.rwLock.Unlock()
-
-	if m.appendLogs == nil {
-		m.appendLogs = make(map[uint64]*entity.SysLog)
-	}
-
-	syslog := m.appendLogs[logId]
+	syslog := m.GetCacheLog(logId)
 	if syslog == nil {
 		syslog = new(entity.SysLog)
 		if err := m.SyslogRepo.GetById(syslog, logId); err != nil {
 			logx.Warnf("追加日志不存在: %d", logId)
 			return
 		}
-		m.appendLogs[logId] = syslog
 	}
 
 	appendLogMsg := fmt.Sprintf("%s %s", timex.DefaultFormat(time.Now()), appendLog.AppendResp)
 	syslog.Resp = fmt.Sprintf("%s\n%s", syslog.Resp, appendLogMsg)
 	syslog.Type = appendLog.Type
-	if appendLog.Extra != nil {
+	if len(appendLog.Extra) > 0 {
 		existExtra := jsonx.ToMap(syslog.Extra)
 		syslog.Extra = jsonx.ToStr(collx.MapMerge(existExtra, appendLog.Extra))
 	}
+
+	m.SetCacheLog(logId, syslog)
 }
 
-func (m *syslogAppImpl) Flush(logId uint64) {
-	syslog := m.appendLogs[logId]
+func (m *syslogAppImpl) SetExtra(logId uint64, key string, val any) {
+	syslog := m.GetCacheLog(logId)
+	if syslog == nil {
+		syslog = new(entity.SysLog)
+		if err := m.SyslogRepo.GetById(syslog, logId); err != nil {
+			logx.Warnf("追加日志不存在: %d", logId)
+			return
+		}
+	}
+
+	extraMap := jsonx.ToMap(syslog.Extra)
+	if extraMap == nil {
+		extraMap = make(map[string]any)
+	}
+	if anyx.IsBlank(val) {
+		delete(extraMap, key)
+	} else {
+		extraMap[key] = val
+	}
+	syslog.Extra = jsonx.ToStr(extraMap)
+
+	m.SetCacheLog(logId, syslog)
+}
+
+func (m *syslogAppImpl) Flush(logId uint64, clearExtra bool) {
+	syslog := m.GetCacheLog(logId)
 	if syslog == nil {
 		return
 	}
@@ -175,6 +190,29 @@ func (m *syslogAppImpl) Flush(logId uint64) {
 		syslog.Type = entity.SyslogTypeSuccess
 	}
 
+	if clearExtra {
+		syslog.Extra = ""
+	}
 	m.SyslogRepo.UpdateById(context.Background(), syslog)
-	delete(m.appendLogs, logId)
+	m.DelCacheLog(logId)
+}
+
+func (m *syslogAppImpl) GetCacheLog(logId uint64) *entity.SysLog {
+	log := new(entity.SysLog)
+	if !cache.Get(getLogKey(logId), log) {
+		return nil
+	}
+	return log
+}
+
+func (m *syslogAppImpl) SetCacheLog(logId uint64, log *entity.SysLog) {
+	cache.Set(getLogKey(logId), log, time.Hour*1)
+}
+
+func (m *syslogAppImpl) DelCacheLog(logId uint64) {
+	cache.Del(getLogKey(logId))
+}
+
+func getLogKey(logId uint64) string {
+	return fmt.Sprintf("mayfly:syslog:%d", logId)
 }

server/pkg/cache/str_cache.go

@@ -1,8 +1,10 @@
 package cache
 
 import (
+	"encoding/json"
 	"mayfly-go/pkg/logx"
 	"mayfly-go/pkg/rediscli"
+	"mayfly-go/pkg/utils/anyx"
 	"strconv"
 	"time"
 )
@@ -39,6 +41,19 @@ func GetInt(key string) int {
 	}
 }
 
+// Get 获取缓存值，并使用json反序列化。返回是否获取成功。若不存在或者解析失败，则返回false
+func Get[T any](key string, valPtr T) bool {
+	strVal := GetStr(key)
+	if strVal == "" {
+		return false
+	}
+	if err := json.Unmarshal([]byte(strVal), valPtr); err != nil {
+		logx.Errorf("json转换缓存中的值失败: %v", err)
+		return false
+	}
+	return true
+}
+
 // 如果系统有设置redis信息，则使用redis存，否则存于本机内存。duration == -1则为永久缓存
 func SetStr(key, value string, duration time.Duration) error {
 	if !UseRedisCache() {
@@ -48,6 +63,16 @@ func SetStr(key, value string, duration time.Duration) error {
 	return rediscli.Set(key, value, duration)
 }
 
+// 如果系统有设置redis信息，则使用redis存，否则存于本机内存。duration == -1则为永久缓存
+func Set(key string, value any, duration time.Duration) error {
+	strVal := anyx.ToString(value)
+	if !UseRedisCache() {
+		checkCache()
+		return tm.Add(key, strVal, duration)
+	}
+	return rediscli.Set(key, strVal, duration)
+}
+
 // 删除指定key
 func Del(key string) {
 	if !UseRedisCache() {