refactor: code review

mayfly_go_web/package.json

@@ -55,7 +55,7 @@
     "prettier": "^3.1.0",
     "sass": "^1.69.0",
     "typescript": "^5.3.2",
-    "vite": "^5.0.10",
+    "vite": "^5.0.11",
     "vue-eslint-parser": "^9.3.2"
   },
   "browserslist": [

mayfly_go_web/src/views/ops/db/SyncTaskEdit.vue

@@ -171,7 +171,7 @@ const basicFormData = {
     srcDbId: -1,
     targetDbId: -1,
     dataSql: 'select * from',
-    pageSize: 100,
+    pageSize: 1000,
     updField: 'id',
     updFieldVal: '0',
     fieldMap: [{ src: 'a', target: 'b' }],
@@ -291,7 +291,7 @@ watch(tabActiveName, async (newValue: string) => {
             let updField = srcDbDialect.wrapName(state.form.updField!);
             state.previewDataSql = `SELECT * FROM (\n ${state.form.dataSql?.trim() || '请输入数据sql'} \n ) t \n where ${updField} > '${
                 state.form.updFieldVal || ''
-            }' \n ${srcDbDialect.getPageSql(1, state.form.pageSize || 100)}`;
+            }'`;
 
             // 检查字段映射中是否存在重复的目标字段
             let fields = new Set();

mayfly_go_web/src/views/ops/db/SyncTaskList.vue

@@ -13,11 +13,6 @@
                 <el-button v-auth="perms.save" type="primary" icon="plus" @click="edit(false)">添加</el-button>
                 <el-button v-auth="perms.del" :disabled="selectionData.length < 1" @click="del()" type="danger" icon="delete">删除</el-button>
             </template>
-            <template #recentState="{ data }">
-                <el-tag v-if="data.recentState == 1" class="ml-2" type="success">成功</el-tag>
-                <el-tag v-else-if="data.recentState == 2" class="ml-2" type="success">失败</el-tag>
-                <el-tag v-else-if="data.recentState == 0" class="ml-2" type="">未执行</el-tag>
-            </template>
             <template #status="{ data }">
                 <span v-if="actionBtns[perms.status]">
                     <el-switch
@@ -59,6 +54,7 @@ import PageTable from '@/components/pagetable/PageTable.vue';
 import { TableColumn } from '@/components/pagetable';
 import { hasPerms } from '@/components/auth/auth';
 import { SearchItem } from '@/components/SearchForm';
+import { DbDataSyncRecentStateEnum, DbDataSyncRunningStateEnum } from './enums';
 
 const DataSyncTaskEdit = defineAsyncComponent(() => import('./SyncTaskEdit.vue'));
 const DataSyncTaskLog = defineAsyncComponent(() => import('./SyncTaskLog.vue'));
@@ -75,7 +71,8 @@ const searchItems = [SearchItem.input('name', '名称')];
 // 任务名、修改人、修改时间、最近一次任务执行状态、状态(停用启用)、操作
 const columns = ref([
     TableColumn.new('taskName', '任务名'),
-    TableColumn.new('recentState', '最近任务状态').alignCenter().isSlot(),
+    TableColumn.new('runningState', '运行状态').alignCenter().typeTag(DbDataSyncRunningStateEnum),
+    TableColumn.new('recentState', '最近任务状态').alignCenter().typeTag(DbDataSyncRecentStateEnum),
     TableColumn.new('status', '状态').alignCenter().isSlot(),
     TableColumn.new('modifier', '修改人').alignCenter(),
     TableColumn.new('updateTime', '修改时间').alignCenter().isTime(),

mayfly_go_web/src/views/ops/db/SyncTaskLog.vue

@@ -23,9 +23,6 @@ import PageTable from '@/components/pagetable/PageTable.vue';
 import { TableColumn } from '@/components/pagetable';
 
 const props = defineProps({
-    visible: {
-        type: Boolean,
-    },
     taskId: {
         type: Number,
     },
@@ -35,6 +32,8 @@ const props = defineProps({
     },
 });
 
+const dialogVisible = defineModel<boolean>('visible', { default: false });
+
 const columns = ref([
     // 状态:1.成功  -1.失败
     TableColumn.new('status', '状态').alignCenter().isSlot(),
@@ -44,14 +43,15 @@ const columns = ref([
     TableColumn.new('resNum', '数据条数'),
 ]);
 
-watch(props, async (newValue: any) => {
+watch(dialogVisible, (newValue: any) => {
-    state.dialogVisible = newValue.visible;
+    if (!newValue) {
-    if (!state.dialogVisible) {
         state.polling = false;
         watchPolling(false);
         return;
     }
+
     state.query.taskId = props.taskId!;
+    search();
     state.realTime = props.running;
     watchPolling(props.running);
 });
@@ -77,14 +77,6 @@ const watchPolling = (polling: boolean) => {
     }
 };
 
-watch(
-    () => props.taskId,
-    async (newValue: any) => {
-        state.query.taskId = newValue!;
-        search();
-    }
-);
-
 const logTableRef: Ref<any> = ref(null);
 
 const search = () => {
@@ -98,13 +90,12 @@ const search = () => {
 const emit = defineEmits(['update:visible', 'cancel', 'val-change']);
 //定义事件
 const cancel = () => {
-    emit('update:visible', false);
+    dialogVisible.value = false;
     emit('cancel');
     watchPolling(false);
 };
 
 const state = reactive({
-    dialogVisible: false,
     polling: false,
     pollingIndex: 0 as any,
     realTime: props.running,
@@ -119,5 +110,5 @@ const state = reactive({
     },
 });
 
-const { dialogVisible, query, realTime } = toRefs(state);
+const { query, realTime } = toRefs(state);
 </script>

mayfly_go_web/src/views/ops/db/enums.ts

@@ -8,3 +8,14 @@ export const DbSqlExecTypeEnum = {
     Query: EnumValue.of(4, 'QUERY').setTagColor('#A8DEE0'),
     Other: EnumValue.of(-1, 'OTHER').setTagColor('#F9E2AE'),
 };
+
+export const DbDataSyncRecentStateEnum = {
+    Success: EnumValue.of(1, '成功').setTagType('success'),
+    Fail: EnumValue.of(-1, '失败').setTagType('danger'),
+};
+
+export const DbDataSyncRunningStateEnum = {
+    Success: EnumValue.of(1, '运行中').setTagType('success'),
+    Wait: EnumValue.of(2, '待运行').setTagType('primary'),
+    Fail: EnumValue.of(3, '已停止').setTagType('danger'),
+};

server/internal/db/api/db.go

@@ -351,7 +351,7 @@ func (d *Db) dumpDb(writer *gzipWriter, dbId uint64, dbName string, tables []str
 		writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表记录: %s \n-- ----------------------------\n", table))
 		writer.WriteString("BEGIN;\n")
 		insertSql := "INSERT INTO %s VALUES (%s);\n"
-		dbMeta.WalkTableRecord(table, func(record map[string]any, columns []*dbm.QueryColumn) {
+		dbMeta.WalkTableRecord(table, func(record map[string]any, columns []*dbm.QueryColumn) error {
 			var values []string
 			writer.TryFlush()
 			for _, column := range columns {
@@ -369,6 +369,7 @@ func (d *Db) dumpDb(writer *gzipWriter, dbId uint64, dbName string, tables []str
 				}
 			}
 			writer.WriteString(fmt.Sprintf(insertSql, quotedTable, strings.Join(values, ", ")))
+			return nil
 		})
 		writer.WriteString("COMMIT;\n")
 	}

server/internal/db/application/db_data_sync.go

@@ -2,17 +2,18 @@ package application
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"mayfly-go/internal/db/dbm"
 	"mayfly-go/internal/db/domain/entity"
 	"mayfly-go/internal/db/domain/repository"
 	"mayfly-go/pkg/base"
+	"mayfly-go/pkg/errorx"
 	"mayfly-go/pkg/gormx"
 	"mayfly-go/pkg/logx"
 	"mayfly-go/pkg/model"
 	"mayfly-go/pkg/scheduler"
-	"strings"
 	"time"
 )
 
@@ -38,7 +39,7 @@ type DataSyncTask interface {
 
 	RemoveCronJobByKey(taskKey string)
 
-	RunCronJob(id uint64)
+	RunCronJob(id uint64) error
 
 	GetTaskLogList(condition *entity.DataSyncLogQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
 }
@@ -80,7 +81,9 @@ func (app *dataSyncAppImpl) AddCronJob(taskEntity *entity.DataSyncTask) {
 	// 根据状态添加新的任务
 	if taskEntity.Status == entity.DataSyncTaskStatusEnable {
 		scheduler.AddFunByKey(key, taskEntity.TaskCron, func() {
-			go app.RunCronJob(taskEntity.Id)
+			if err := app.RunCronJob(taskEntity.Id); err != nil {
+				logx.Errorf("定时执行数据同步任务失败: %s", err.Error())
+			}
 		})
 	}
 }
@@ -120,51 +123,21 @@ func (app *dataSyncAppImpl) changeRunningState(id uint64, state int8) {
 	_ = app.UpdateById(context.Background(), task)
 }
 
-func (app *dataSyncAppImpl) RunCronJob(id uint64) {
+func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
 	// 查询最新的任务信息
 	task, err := app.GetById(new(entity.DataSyncTask), id)
 	if err != nil {
-		logx.Warnf("[%d]任务不存在", id)
+		return errorx.NewBiz("任务不存在")
-		return
 	}
 	if task.RunningState == entity.DataSyncTaskRunStateRunning {
-		logx.Warnf("数据同步任务正在执行中：%s => %s", task.TaskName, task.TaskKey)
+		return errorx.NewBiz("该任务正在执行中")
-		return
 	}
 	// 开始运行时，修改状态为运行中
 	app.changeRunningState(id, entity.DataSyncTaskRunStateRunning)
 
-	logx.Warnf("开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
+	logx.Infof("开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
 
-	// 获取源数据库连接
+	go func() {
-	srcConn, err := GetDbApp().GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
-	if err != nil {
-		app.endRunning(task, entity.DataSyncTaskStateFail, "连接源数据库失败", "", 0)
-		return
-	}
-
-	// 获取目标数据库连接
-	targetConn, err := GetDbApp().GetDbConn(uint64(task.TargetDbId), task.TargetDbName)
-	if err != nil {
-		app.endRunning(task, entity.DataSyncTaskStateFail, "连接目标数据库失败", "", 0)
-		return
-	}
-
-	// 当前分页
-	page := 1
-	// 记录每次分页返回数据条数
-	resSize := task.PageSize
-	// 记录本次同步数据总数
-	total := 0
-	srcDialect := srcConn.GetDialect()
-	// 记录更新字段最新值
-	updFieldVal := task.UpdFieldVal
-	targetDialect := targetConn.GetDialect()
-
-	for {
-		if resSize < task.PageSize {
-			break
-		}
 		// 通过占位符格式化sql
 		updSql := ""
 		orderSql := ""
@@ -172,140 +145,174 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) {
 			updSql = fmt.Sprintf("and %s > '%s'", task.UpdField, task.UpdFieldVal)
 			orderSql = "order by " + task.UpdField + " asc "
 		}
-
-		pageSql := srcDialect.PageSql(page, task.PageSize)
 		// 组装查询sql
-		sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s %s", task.DataSql, updSql, orderSql, pageSql)
+		sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s", task.DataSql, updSql, orderSql)
-		logx.Infof("同步任务：[%s]，执行sql：[%s]", task.TaskName, sql)
+
-		// 源数据库执行sql查询结果
+		err = app.doDataSync(sql, task)
-		columns, res, err := srcConn.Query(sql)
 		if err != nil {
-			app.endRunning(task, entity.DataSyncTaskStateFail, fmt.Sprintf("查询源数据库失败:%s", err.Error()), sql, 0)
+			app.endRunning(task, entity.DataSyncTaskStateFail, fmt.Sprintf("执行失败: %s", err.Error()), sql, 0)
-			return
 		}
-		if len(res) == 0 {
+	}()
-			app.endRunning(task, entity.DataSyncTaskStateSuccess, fmt.Sprintf("执行成功，新数据：%d 条", total), sql, 0)
-			return
-		}
-		// 每次分页查询成功后，记录一些数据
-		resSize = len(res)
-		total += resSize
-		page++
-		index := 0
 
-		// task.FieldMap为json数组字符串 [{"src":"id","target":"id"}]，转为map
+	return nil
-		var fieldMap []map[string]string
+}
-		err = json.Unmarshal([]byte(task.FieldMap), &fieldMap)
-		if err != nil {
-			app.endRunning(task, entity.DataSyncTaskStateFail, "解析字段映射json出错", sql, resSize)
-			return
-		}
 
-		// 遍历columns 取task.UpdField的字段类型
+func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) error {
-		updFieldType := dbm.DataTypeString
+	// 获取源数据库连接
-		for _, column := range columns {
+	srcConn, err := GetDbApp().GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
-			if column.Name == task.UpdField {
+	if err != nil {
-				updFieldType = srcDialect.GetDataType(column.Type)
+		return errorx.NewBiz("连接源数据库失败: %s", err.Error())
-				break
-			}
-		}
-
-		var data = make([]map[string]any, 0)
-
-		// 遍历res，组装插入sql
-		for _, record := range res {
-			index++
-			// 获取查询结果最后一条数据的UpdField字段值
-			if index == resSize {
-				updFieldVal = fmt.Sprintf("%v", record[task.UpdField])
-				updFieldVal = srcDialect.FormatStrData(updFieldVal, updFieldType)
-			}
-			var rowData = make(map[string]any)
-			// 遍历字段映射, target字段的值为src字段取值
-			for _, item := range fieldMap {
-				srcField := item["src"]
-				targetField := item["target"]
-				// target字段的值为src字段取值
-				rowData[targetField] = record[srcField]
-			}
-
-			data = append(data, rowData)
-		}
-
-		// 获取目标库字段数组
-		targetWrapColumns := make([]string, 0)
-		// 获取源库字段数组
-		srcColumns := make([]string, 0)
-		for _, item := range fieldMap {
-			targetField := item["target"]
-			srcField := item["target"]
-			targetWrapColumns = append(targetWrapColumns, targetDialect.WrapName(targetField))
-			srcColumns = append(srcColumns, srcField)
-		}
-
-		// 从目标库数据中取出源库字段对应的值
-		values := make([][]any, 0)
-		for _, record := range data {
-			rawValue := make([]any, 0)
-			for _, column := range srcColumns {
-				rawValue = append(rawValue, record[column])
-			}
-			values = append(values, rawValue)
-		}
-
-		// 生成占位符字符串：如：(?,?)
-		// 重复字符串并用逗号连接
-		repeated := strings.Repeat("?,", len(targetWrapColumns))
-		// 去除最后一个逗号，占位符由括号包裹
-		placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
-
-		// 目标数据库执行sql批量插入
-		err = targetDialect.SaveBatch(targetConn, task.TargetTableName, strings.Join(targetWrapColumns, ","), placeholder, values)
-		if err != nil {
-			// 保存执行成功日志
-			logx.Errorf("保存记录失败：%s", err.Error())
-			app.endRunning(task, entity.DataSyncTaskStateFail, err.Error(), sql, resSize)
-			return
-		}
-
-		// 保存运行时日志
-		logx.Infof("同步任务：[%s],保存记录成功：[%d]条", task.TaskName, total)
-		app.saveLog(task.Id, entity.DataSyncTaskStateSuccess, fmt.Sprintf("分页执行成功，新数据：%d 条", total), sql, total)
-
-		// 运行过程中，判断状态是否为已关闭，是则结束运行，否则继续运行
-		taskParam, _ := app.GetById(new(entity.DataSyncTask), id)
-		if taskParam.RunningState == entity.DataSyncTaskRunStateStop {
-			app.endRunning(task, entity.DataSyncTaskStateFail, "手动停止任务", sql, resSize)
-			return
-		}
-
-		// 记录一次数据状态
-		taskParam = new(entity.DataSyncTask)
-		taskParam.Id = task.Id
-		taskParam.UpdFieldVal = updFieldVal
-		taskParam.RecentState = entity.DataSyncTaskStateSuccess
-		taskParam.RunningState = entity.DataSyncTaskRunStateRunning
-		_ = app.UpdateById(context.Background(), taskParam)
 	}
 
-	logx.Infof("同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
+	// 获取目标数据库连接
+	targetConn, err := GetDbApp().GetDbConn(uint64(task.TargetDbId), task.TargetDbName)
+	if err != nil {
+		return errorx.NewBiz("连接目标数据库失败: %s", err.Error())
+	}
+	targetDbTx, err := targetConn.Begin()
+	if err != nil {
+		return errorx.NewBiz("开启目标数据库事务失败: %s", err.Error())
+	}
+	defer func() {
+		if r := recover(); r != nil {
+			targetDbTx.Rollback()
+			err = fmt.Errorf("%v", r)
+		}
+	}()
 
+	srcDialect := srcConn.GetDialect()
 	// 记录更新字段最新值
-	task.UpdFieldVal = updFieldVal
+	targetDialect := targetConn.GetDialect()
+
+	// task.FieldMap为json数组字符串 [{"src":"id","target":"id"}]，转为map
+	var fieldMap []map[string]string
+	err = json.Unmarshal([]byte(task.FieldMap), &fieldMap)
+	if err != nil {
+		return errorx.NewBiz("解析字段映射json出错: %s", err.Error())
+	}
+	var updFieldType dbm.DataType
+
+	// 记录本次同步数据总数
+	total := 0
+	batchSize := task.PageSize
+	result := make([]map[string]any, 0)
+	var queryColumns []*dbm.QueryColumn
+
+	err = srcConn.WalkQueryRows(context.Background(), sql, func(row map[string]any, columns []*dbm.QueryColumn) error {
+		if len(queryColumns) == 0 {
+			queryColumns = columns
+
+			// 遍历columns 取task.UpdField的字段类型
+			updFieldType = dbm.DataTypeString
+			for _, column := range columns {
+				if column.Name == task.UpdField {
+					updFieldType = srcDialect.GetDataType(column.Type)
+					break
+				}
+			}
+		}
+
+		total++
+		result = append(result, row)
+		if total%batchSize == 0 {
+			if err := app.srcData2TargetDb(result, fieldMap, updFieldType, task, srcDialect, targetDialect, targetDbTx); err != nil {
+				return err
+			}
+			result = result[:0]
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		targetDbTx.Rollback()
+		return err
+	}
+
+	// 处理剩余的数据
+	if len(result) > 0 {
+		if err := app.srcData2TargetDb(result, fieldMap, updFieldType, task, srcDialect, targetDialect, targetDbTx); err != nil {
+			targetDbTx.Rollback()
+			return err
+		}
+	}
+
+	if err := targetDbTx.Commit(); err != nil {
+		return errorx.NewBiz("数据同步-目标数据库事务提交失败: %s", err.Error())
+	}
+	logx.Infof("同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
 
 	// 保存执行成功日志
 	app.endRunning(task, entity.DataSyncTaskStateSuccess, fmt.Sprintf("本次任务执行成功，新数据：%d 条", total), "", total)
+	return nil
+}
 
+func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, updFieldType dbm.DataType, task *entity.DataSyncTask, srcDialect dbm.DbDialect, targetDialect dbm.DbDialect, targetDbTx *sql.Tx) error {
+	var data = make([]map[string]any, 0)
+
+	// 遍历res，组装插入sql
+	for _, record := range srcRes {
+		var rowData = make(map[string]any)
+		// 遍历字段映射, target字段的值为src字段取值
+		for _, item := range fieldMap {
+			srcField := item["src"]
+			targetField := item["target"]
+			// target字段的值为src字段取值
+			rowData[targetField] = record[srcField]
+		}
+
+		data = append(data, rowData)
+	}
+
+	updFieldVal := fmt.Sprintf("%v", srcRes[len(srcRes)-1][task.UpdField])
+	updFieldVal = srcDialect.FormatStrData(updFieldVal, updFieldType)
+	task.UpdFieldVal = updFieldVal
+
+	// 获取目标库字段数组
+	targetWrapColumns := make([]string, 0)
+	// 获取源库字段数组
+	srcColumns := make([]string, 0)
+	for _, item := range fieldMap {
+		targetField := item["target"]
+		srcField := item["target"]
+		targetWrapColumns = append(targetWrapColumns, targetDialect.WrapName(targetField))
+		srcColumns = append(srcColumns, srcField)
+	}
+
+	// 从目标库数据中取出源库字段对应的值
+	values := make([][]any, 0)
+	for _, record := range data {
+		rawValue := make([]any, 0)
+		for _, column := range srcColumns {
+			rawValue = append(rawValue, record[column])
+		}
+		values = append(values, rawValue)
+	}
+
+	// 目标数据库执行sql批量插入
+	_, err := targetDialect.BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values)
+	if err != nil {
+		return err
+	}
+
+	// 运行过程中，判断状态是否为已关闭，是则结束运行，否则继续运行
+	taskParam, _ := app.GetById(new(entity.DataSyncTask), task.Id)
+	if taskParam.RunningState == entity.DataSyncTaskRunStateStop {
+		return errorx.NewBiz("该任务已被手动终止")
+	}
+
+	return nil
 }
 
 func (app *dataSyncAppImpl) endRunning(taskEntity *entity.DataSyncTask, state int8, msg string, sql string, resNum int) {
-
 	logx.Info(msg)
 
 	task := new(entity.DataSyncTask)
 	task.Id = taskEntity.Id
 	task.RecentState = state
-	task.UpdFieldVal = taskEntity.UpdFieldVal
+	if state == entity.DataSyncTaskStateSuccess {
+		task.UpdFieldVal = taskEntity.UpdFieldVal
+	}
 	task.RunningState = entity.DataSyncTaskRunStateReady
 	// 运行失败之后设置任务状态为禁用
 	//if state == entity.DataSyncTaskStateFail {
@@ -315,7 +322,6 @@ func (app *dataSyncAppImpl) endRunning(taskEntity *entity.DataSyncTask, state in
 	_ = app.UpdateById(context.Background(), task)
 	// 保存执行日志
 	app.saveLog(taskEntity.Id, state, msg, sql, resNum)
-
 }
 
 func (app *dataSyncAppImpl) saveLog(taskId uint64, state int8, msg string, sql string, resNum int) {

server/internal/db/dbm/conn.go

@@ -12,6 +12,9 @@ import (
 	"strings"
 )
 
+// 游标遍历查询结果集处理函数
+type WalkQueryRowsFunc func(row map[string]any, columns []*QueryColumn) error
+
 // db实例连接信息
 type DbConn struct {
 	Id   string
@@ -36,13 +39,21 @@ func (d *DbConn) Query(querySql string, args ...any) ([]*QueryColumn, []map[stri
 // 依次返回 列信息数组(顺序)，结果map，错误
 func (d *DbConn) QueryContext(ctx context.Context, querySql string, args ...any) ([]*QueryColumn, []map[string]any, error) {
 	result := make([]map[string]any, 0, 16)
-	columns, err := walkTableRecord(ctx, d.db, querySql, func(record map[string]any, columns []*QueryColumn) {
+	var queryColumns []*QueryColumn
-		result = append(result, record)
+
+	err := d.WalkQueryRows(ctx, querySql, func(row map[string]any, columns []*QueryColumn) error {
+		if len(queryColumns) == 0 {
+			queryColumns = columns
+		}
+		result = append(result, row)
+		return nil
 	}, args...)
+
 	if err != nil {
 		return nil, nil, wrapSqlError(err)
 	}
-	return columns, result, nil
+
+	return queryColumns, result, nil
 }
 
 // 将查询结果映射至struct，可具体参考sqlx库
@@ -61,10 +72,9 @@ func (d *DbConn) Query2Struct(execSql string, dest any) error {
 	return scanAll(rows, dest, false)
 }
 
-// WalkTableRecord 遍历表记录
+// 游标方式遍历查询结果集, walkFn返回error不为nil, 则跳出遍历
-func (d *DbConn) WalkTableRecord(ctx context.Context, selectSql string, walk func(record map[string]any, columns []*QueryColumn)) error {
+func (d *DbConn) WalkQueryRows(ctx context.Context, querySql string, walkFn WalkQueryRowsFunc, args ...any) error {
-	_, err := walkTableRecord(ctx, d.db, selectSql, walk)
+	return walkQueryRows(ctx, d.db, querySql, walkFn, args...)
-	return err
 }
 
 // 执行 update, insert, delete，建表等sql
@@ -73,16 +83,44 @@ func (d *DbConn) Exec(sql string, args ...any) (int64, error) {
 	return d.ExecContext(context.Background(), sql, args...)
 }
 
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不使用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExec(tx *sql.Tx, execSql string, args ...any) (int64, error) {
+	return d.TxExecContext(context.Background(), tx, execSql, args...)
+}
+
 // 执行 update, insert, delete，建表等sql
 // 返回影响条数和错误
-func (d *DbConn) ExecContext(ctx context.Context, sql string, args ...any) (int64, error) {
+func (d *DbConn) ExecContext(ctx context.Context, execSql string, args ...any) (int64, error) {
-	res, err := d.db.ExecContext(ctx, sql, args...)
+	res, err := d.db.ExecContext(ctx, execSql, args...)
 	if err != nil {
 		return 0, wrapSqlError(err)
 	}
 	return res.RowsAffected()
 }
 
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不适用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExecContext(ctx context.Context, tx *sql.Tx, execSql string, args ...any) (int64, error) {
+	var res sql.Result
+	var err error
+	if tx != nil {
+		res, err = tx.ExecContext(ctx, execSql, args...)
+	} else {
+		res, err = d.db.ExecContext(ctx, execSql, args...)
+	}
+
+	if err != nil {
+		return 0, wrapSqlError(err)
+	}
+	return res.RowsAffected()
+}
+
+// 开启事务
+func (d *DbConn) Begin() (*sql.Tx, error) {
+	return d.db.Begin()
+}
+
 // 获取数据库元信息实现接口
 func (d *DbConn) GetDialect() DbDialect {
 	switch d.Info.Type {
@@ -111,11 +149,12 @@ func (d *DbConn) Close() {
 	}
 }
 
-func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk func(record map[string]any, columns []*QueryColumn), args ...any) ([]*QueryColumn, error) {
+// 游标方式遍历查询rows, walkFn error不为nil, 则跳出遍历
+func walkQueryRows(ctx context.Context, db *sql.DB, selectSql string, walkFn WalkQueryRowsFunc, args ...any) error {
 	rows, err := db.QueryContext(ctx, selectSql, args...)
 
 	if err != nil {
-		return nil, err
+		return err
 	}
 	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
 	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
@@ -127,7 +166,7 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
 
 	colTypes, err := rows.ColumnTypes()
 	if err != nil {
-		return nil, err
+		return err
 	}
 	lenCols := len(colTypes)
 	// 列名用于前端表头名称按照数据库与查询字段顺序显示
@@ -145,7 +184,7 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
 	for rows.Next() {
 		// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
 		if err := rows.Scan(scans...); err != nil {
-			return nil, err
+			return err
 		}
 		// 每行数据
 		rowData := make(map[string]any, lenCols)
@@ -153,10 +192,13 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
 		for i, v := range values {
 			rowData[colTypes[i].Name()] = valueConvert(v, colTypes[i])
 		}
-		walk(rowData, cols)
+		if err = walkFn(rowData, cols); err != nil {
+			logx.Error("游标遍历查询结果集出错,退出遍历: %s", err.Error())
+			return err
+		}
 	}
 
-	return cols, nil
+	return nil
 }
 
 // 将查询的值转为对应列类型的实际值，不全部转为字符串

server/internal/db/dbm/dialect.go

@@ -1,6 +1,7 @@
 package dbm
 
 import (
+	"database/sql"
 	"embed"
 	"mayfly-go/pkg/biz"
 	"mayfly-go/pkg/utils/collx"
@@ -71,12 +72,8 @@ type DbDialect interface {
 	// 获取建表ddl
 	GetTableDDL(tableName string) (string, error)
 
-	// 获取指定表的数据-分页查询
-	// @return columns: 列字段名；result: 结果集；error: 错误
-	GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error)
-
 	// WalkTableRecord 遍历指定表的数据
-	WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error
+	WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error
 
 	GetSchemas() ([]string, error)
 
@@ -86,11 +83,8 @@ type DbDialect interface {
 	// 封装名字，如，mysql: `table_name`,  dm: "table_name"
 	WrapName(name string) string
 
-	// 分页sql，如：mysql: limit 1 ,10, dm: limit 10 offset 1
-	PageSql(pageNum int, pageSize int) string
-
 	// 批量保存数据
-	SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error
+	BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error)
 
 	GetDataType(dbColumnType string) DataType
 

server/internal/db/dbm/dialect_dm.go

@@ -255,12 +255,8 @@ func (dd *DMDialect) GetTableDDL(tableName string) (string, error) {
 	return builder.String(), nil
 }
 
-func (dd *DMDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
+func (dd *DMDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
-	return dd.dc.Query(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
+	return dd.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
-}
-
-func (dd *DMDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
-	return dd.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
 }
 
 // 获取DM当前连接的库可访问的schemaNames
@@ -286,10 +282,6 @@ func (pd *DMDialect) WrapName(name string) string {
 	return "\"" + name + "\""
 }
 
-func (pd *DMDialect) PageSql(pageNum int, pageSize int) string {
-	return fmt.Sprintf("LIMIT %d OFFSET %d", pageSize, (pageNum-1)*pageSize)
-}
-
 func (pd *DMDialect) GetDataType(dbColumnType string) DataType {
 	if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
 		return DataTypeNumber
@@ -309,21 +301,29 @@ func (pd *DMDialect) GetDataType(dbColumnType string) DataType {
 	return DataTypeString
 }
 
-func (pd *DMDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
 	// 执行批量insert sql
 	// insert into "table_name" ("column1", "column2", ...) values (value1, value2, ...)
 
-	sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+	// 生成占位符字符串：如：(?,?)
+	// 重复字符串并用逗号连接
+	repeated := strings.Repeat("?,", len(columns))
+	// 去除最后一个逗号，占位符由括号包裹
+	placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
+	sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
+	effRows := 0
 	for _, value := range values {
 		// 达梦数据库只能一条条的执行insert
-		_, err := conn.Exec(sqlTemp, value...)
+		er, err := pd.dc.TxExec(tx, sqlTemp, value...)
 		if err != nil {
 			logx.Errorf("执行sql失败：%s", err.Error())
-			return err
+			return int64(effRows), err
 		}
+		effRows += int(er)
 	}
 	// 执行批量insert sql
-	return nil
+	return int64(effRows), nil
 }
 
 func (pd *DMDialect) FormatStrData(dbColumnValue string, dataType DataType) string {

server/internal/db/dbm/dialect_mysql.go

@@ -189,12 +189,8 @@ func (md *MysqlDialect) GetTableDDL(tableName string) (string, error) {
 	return res[0]["Create Table"].(string) + ";", nil
 }
 
-func (md *MysqlDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
+func (md *MysqlDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
-	return md.dc.Query(fmt.Sprintf("SELECT * FROM %s LIMIT %d, %d", tableName, (pageNum-1)*pageSize, pageSize))
+	return md.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
-}
-
-func (md *MysqlDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
-	return md.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
 }
 
 func (md *MysqlDialect) GetSchemas() ([]string, error) {
@@ -210,10 +206,6 @@ func (pd *MysqlDialect) WrapName(name string) string {
 	return "`" + name + "`"
 }
 
-func (pd *MysqlDialect) PageSql(pageNum int, pageSize int) string {
-	return fmt.Sprintf("limit %d, %d", (pageNum-1)*pageSize, pageSize)
-}
-
 func (pd *MysqlDialect) GetDataType(dbColumnType string) DataType {
 	if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
 		return DataTypeNumber
@@ -233,24 +225,29 @@ func (pd *MysqlDialect) GetDataType(dbColumnType string) DataType {
 	return DataTypeString
 }
 
-func (pd *MysqlDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
+	// 生成占位符字符串：如：(?,?)
+	// 重复字符串并用逗号连接
+	repeated := strings.Repeat("?,", len(columns))
+	// 去除最后一个逗号，占位符由括号包裹
+	placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
 	// 执行批量insert sql，mysql支持批量insert语法
 	// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
 
 	// 重复占位符字符串n遍
-	repeated := strings.Repeat(placeholder+",", len(values))
+	repeated = strings.Repeat(placeholder+",", len(values))
 	// 去除最后一个逗号
 	placeholder = strings.TrimSuffix(repeated, ",")
 
-	sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+	sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
 	// 执行批量insert sql
 	// 把二维数组转为一维数组
 	var args []any
 	for _, v := range values {
 		args = append(args, v...)
 	}
-	_, err := conn.Exec(sqlStr, args...)
+	return pd.dc.TxExec(tx, sqlStr, args...)
-	return err
 }
 
 func (pd *MysqlDialect) FormatStrData(dbColumnValue string, dataType DataType) string {

server/internal/db/dbm/dialect_pgsql.go

@@ -257,12 +257,8 @@ func (pd *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
 	return res[0]["sql"].(string), nil
 }
 
-func (pd *PgsqlDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
+func (pd *PgsqlDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
-	return pd.dc.Query(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
+	return pd.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
-}
-
-func (pd *PgsqlDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
-	return pd.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
 }
 
 // 获取pgsql当前连接的库可访问的schemaNames
@@ -288,10 +284,6 @@ func (pd *PgsqlDialect) WrapName(name string) string {
 	return name
 }
 
-func (pd *PgsqlDialect) PageSql(pageNum int, pageSize int) string {
-	return fmt.Sprintf("LIMIT %d OFFSET %d", pageSize, (pageNum-1)*pageSize)
-}
-
 func (pd *PgsqlDialect) GetDataType(dbColumnType string) DataType {
 	if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
 		return DataTypeNumber
@@ -311,24 +303,33 @@ func (pd *PgsqlDialect) GetDataType(dbColumnType string) DataType {
 	return DataTypeString
 }
 
-func (pd *PgsqlDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
 	// 执行批量insert sql，跟mysql一样  pg或高斯支持批量insert语法
 	// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
 
+	// 生成占位符字符串：如：(?,?)
+	// 重复字符串并用逗号连接
+	repeated := strings.Repeat("?,", len(columns))
+	// 去除最后一个逗号，占位符由括号包裹
+	placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
+	// 执行批量insert sql，mysql支持批量insert语法
+	// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
+
 	// 重复占位符字符串n遍
-	repeated := strings.Repeat(placeholder+",", len(values))
+	repeated = strings.Repeat(placeholder+",", len(values))
 	// 去除最后一个逗号
 	placeholder = strings.TrimSuffix(repeated, ",")
 
-	sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+	sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
 	// 执行批量insert sql
 	// 把二维数组转为一维数组
 	var args []any
 	for _, v := range values {
 		args = append(args, v...)
 	}
-	_, err := conn.Exec(sqlStr, args...)
+
-	return err
+	return pd.dc.TxExec(tx, sqlStr, args...)
 }
 
 func (pd *PgsqlDialect) FormatStrData(dbColumnValue string, dataType DataType) string {