diff --git a/mayfly_go_web/package.json b/mayfly_go_web/package.json
index 4ac24da8..4673d4b3 100644
--- a/mayfly_go_web/package.json
+++ b/mayfly_go_web/package.json
@@ -55,7 +55,7 @@
"prettier": "^3.1.0",
"sass": "^1.69.0",
"typescript": "^5.3.2",
- "vite": "^5.0.10",
+ "vite": "^5.0.11",
"vue-eslint-parser": "^9.3.2"
},
"browserslist": [
diff --git a/mayfly_go_web/src/views/ops/db/SyncTaskEdit.vue b/mayfly_go_web/src/views/ops/db/SyncTaskEdit.vue
index 915a1597..5962f4cc 100644
--- a/mayfly_go_web/src/views/ops/db/SyncTaskEdit.vue
+++ b/mayfly_go_web/src/views/ops/db/SyncTaskEdit.vue
@@ -171,7 +171,7 @@ const basicFormData = {
srcDbId: -1,
targetDbId: -1,
dataSql: 'select * from',
- pageSize: 100,
+ pageSize: 1000,
updField: 'id',
updFieldVal: '0',
fieldMap: [{ src: 'a', target: 'b' }],
@@ -291,7 +291,7 @@ watch(tabActiveName, async (newValue: string) => {
let updField = srcDbDialect.wrapName(state.form.updField!);
state.previewDataSql = `SELECT * FROM (\n ${state.form.dataSql?.trim() || '请输入数据sql'} \n ) t \n where ${updField} > '${
state.form.updFieldVal || ''
- }' \n ${srcDbDialect.getPageSql(1, state.form.pageSize || 100)}`;
+ }'`;
// 检查字段映射中是否存在重复的目标字段
let fields = new Set();
diff --git a/mayfly_go_web/src/views/ops/db/SyncTaskList.vue b/mayfly_go_web/src/views/ops/db/SyncTaskList.vue
index fbcee029..ae13c7d9 100644
--- a/mayfly_go_web/src/views/ops/db/SyncTaskList.vue
+++ b/mayfly_go_web/src/views/ops/db/SyncTaskList.vue
@@ -13,11 +13,6 @@
添加
删除
-
- 成功
- 失败
- 未执行
-
import('./SyncTaskEdit.vue'));
const DataSyncTaskLog = defineAsyncComponent(() => import('./SyncTaskLog.vue'));
@@ -75,7 +71,8 @@ const searchItems = [SearchItem.input('name', '名称')];
// 任务名、修改人、修改时间、最近一次任务执行状态、状态(停用启用)、操作
const columns = ref([
TableColumn.new('taskName', '任务名'),
- TableColumn.new('recentState', '最近任务状态').alignCenter().isSlot(),
+ TableColumn.new('runningState', '运行状态').alignCenter().typeTag(DbDataSyncRunningStateEnum),
+ TableColumn.new('recentState', '最近任务状态').alignCenter().typeTag(DbDataSyncRecentStateEnum),
TableColumn.new('status', '状态').alignCenter().isSlot(),
TableColumn.new('modifier', '修改人').alignCenter(),
TableColumn.new('updateTime', '修改时间').alignCenter().isTime(),
diff --git a/mayfly_go_web/src/views/ops/db/SyncTaskLog.vue b/mayfly_go_web/src/views/ops/db/SyncTaskLog.vue
index 6596ddfa..e35984aa 100644
--- a/mayfly_go_web/src/views/ops/db/SyncTaskLog.vue
+++ b/mayfly_go_web/src/views/ops/db/SyncTaskLog.vue
@@ -23,9 +23,6 @@ import PageTable from '@/components/pagetable/PageTable.vue';
import { TableColumn } from '@/components/pagetable';
const props = defineProps({
- visible: {
- type: Boolean,
- },
taskId: {
type: Number,
},
@@ -35,6 +32,8 @@ const props = defineProps({
},
});
+const dialogVisible = defineModel('visible', { default: false });
+
const columns = ref([
// 状态:1.成功 -1.失败
TableColumn.new('status', '状态').alignCenter().isSlot(),
@@ -44,14 +43,15 @@ const columns = ref([
TableColumn.new('resNum', '数据条数'),
]);
-watch(props, async (newValue: any) => {
- state.dialogVisible = newValue.visible;
- if (!state.dialogVisible) {
+watch(dialogVisible, (newValue: any) => {
+ if (!newValue) {
state.polling = false;
watchPolling(false);
return;
}
+
state.query.taskId = props.taskId!;
+ search();
state.realTime = props.running;
watchPolling(props.running);
});
@@ -77,14 +77,6 @@ const watchPolling = (polling: boolean) => {
}
};
-watch(
- () => props.taskId,
- async (newValue: any) => {
- state.query.taskId = newValue!;
- search();
- }
-);
-
const logTableRef: Ref = ref(null);
const search = () => {
@@ -98,13 +90,12 @@ const search = () => {
const emit = defineEmits(['update:visible', 'cancel', 'val-change']);
//定义事件
const cancel = () => {
- emit('update:visible', false);
+ dialogVisible.value = false;
emit('cancel');
watchPolling(false);
};
const state = reactive({
- dialogVisible: false,
polling: false,
pollingIndex: 0 as any,
realTime: props.running,
@@ -119,5 +110,5 @@ const state = reactive({
},
});
-const { dialogVisible, query, realTime } = toRefs(state);
+const { query, realTime } = toRefs(state);
diff --git a/mayfly_go_web/src/views/ops/db/enums.ts b/mayfly_go_web/src/views/ops/db/enums.ts
index c8581f91..4e77b289 100644
--- a/mayfly_go_web/src/views/ops/db/enums.ts
+++ b/mayfly_go_web/src/views/ops/db/enums.ts
@@ -8,3 +8,14 @@ export const DbSqlExecTypeEnum = {
Query: EnumValue.of(4, 'QUERY').setTagColor('#A8DEE0'),
Other: EnumValue.of(-1, 'OTHER').setTagColor('#F9E2AE'),
};
+
+export const DbDataSyncRecentStateEnum = {
+ Success: EnumValue.of(1, '成功').setTagType('success'),
+ Fail: EnumValue.of(-1, '失败').setTagType('danger'),
+};
+
+export const DbDataSyncRunningStateEnum = {
+ Success: EnumValue.of(1, '运行中').setTagType('success'),
+ Wait: EnumValue.of(2, '待运行').setTagType('primary'),
+ Fail: EnumValue.of(3, '已停止').setTagType('danger'),
+};
diff --git a/server/internal/db/api/db.go b/server/internal/db/api/db.go
index 0c671add..95f69a83 100644
--- a/server/internal/db/api/db.go
+++ b/server/internal/db/api/db.go
@@ -351,7 +351,7 @@ func (d *Db) dumpDb(writer *gzipWriter, dbId uint64, dbName string, tables []str
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表记录: %s \n-- ----------------------------\n", table))
writer.WriteString("BEGIN;\n")
insertSql := "INSERT INTO %s VALUES (%s);\n"
- dbMeta.WalkTableRecord(table, func(record map[string]any, columns []*dbm.QueryColumn) {
+ dbMeta.WalkTableRecord(table, func(record map[string]any, columns []*dbm.QueryColumn) error {
var values []string
writer.TryFlush()
for _, column := range columns {
@@ -369,6 +369,7 @@ func (d *Db) dumpDb(writer *gzipWriter, dbId uint64, dbName string, tables []str
}
}
writer.WriteString(fmt.Sprintf(insertSql, quotedTable, strings.Join(values, ", ")))
+ return nil
})
writer.WriteString("COMMIT;\n")
}
diff --git a/server/internal/db/application/db_data_sync.go b/server/internal/db/application/db_data_sync.go
index b34592ef..5fa5345f 100644
--- a/server/internal/db/application/db_data_sync.go
+++ b/server/internal/db/application/db_data_sync.go
@@ -2,17 +2,18 @@ package application
import (
"context"
+ "database/sql"
"encoding/json"
"fmt"
"mayfly-go/internal/db/dbm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
+ "mayfly-go/pkg/errorx"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/model"
"mayfly-go/pkg/scheduler"
- "strings"
"time"
)
@@ -38,7 +39,7 @@ type DataSyncTask interface {
RemoveCronJobByKey(taskKey string)
- RunCronJob(id uint64)
+ RunCronJob(id uint64) error
GetTaskLogList(condition *entity.DataSyncLogQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
}
@@ -80,7 +81,9 @@ func (app *dataSyncAppImpl) AddCronJob(taskEntity *entity.DataSyncTask) {
// 根据状态添加新的任务
if taskEntity.Status == entity.DataSyncTaskStatusEnable {
scheduler.AddFunByKey(key, taskEntity.TaskCron, func() {
- go app.RunCronJob(taskEntity.Id)
+ if err := app.RunCronJob(taskEntity.Id); err != nil {
+ logx.Errorf("定时执行数据同步任务失败: %s", err.Error())
+ }
})
}
}
@@ -120,51 +123,21 @@ func (app *dataSyncAppImpl) changeRunningState(id uint64, state int8) {
_ = app.UpdateById(context.Background(), task)
}
-func (app *dataSyncAppImpl) RunCronJob(id uint64) {
+func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
// 查询最新的任务信息
task, err := app.GetById(new(entity.DataSyncTask), id)
if err != nil {
- logx.Warnf("[%d]任务不存在", id)
- return
+ return errorx.NewBiz("任务不存在")
}
if task.RunningState == entity.DataSyncTaskRunStateRunning {
- logx.Warnf("数据同步任务正在执行中：%s => %s", task.TaskName, task.TaskKey)
- return
+ return errorx.NewBiz("该任务正在执行中")
}
// 开始运行时，修改状态为运行中
app.changeRunningState(id, entity.DataSyncTaskRunStateRunning)
- logx.Warnf("开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
+ logx.Infof("开始执行数据同步任务：%s => %s", task.TaskName, task.TaskKey)
- // 获取源数据库连接
- srcConn, err := GetDbApp().GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
- if err != nil {
- app.endRunning(task, entity.DataSyncTaskStateFail, "连接源数据库失败", "", 0)
- return
- }
-
- // 获取目标数据库连接
- targetConn, err := GetDbApp().GetDbConn(uint64(task.TargetDbId), task.TargetDbName)
- if err != nil {
- app.endRunning(task, entity.DataSyncTaskStateFail, "连接目标数据库失败", "", 0)
- return
- }
-
- // 当前分页
- page := 1
- // 记录每次分页返回数据条数
- resSize := task.PageSize
- // 记录本次同步数据总数
- total := 0
- srcDialect := srcConn.GetDialect()
- // 记录更新字段最新值
- updFieldVal := task.UpdFieldVal
- targetDialect := targetConn.GetDialect()
-
- for {
- if resSize < task.PageSize {
- break
- }
+ go func() {
// 通过占位符格式化sql
updSql := ""
orderSql := ""
@@ -172,140 +145,174 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) {
updSql = fmt.Sprintf("and %s > '%s'", task.UpdField, task.UpdFieldVal)
orderSql = "order by " + task.UpdField + " asc "
}
-
- pageSql := srcDialect.PageSql(page, task.PageSize)
// 组装查询sql
- sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s %s", task.DataSql, updSql, orderSql, pageSql)
- logx.Infof("同步任务：[%s]，执行sql：[%s]", task.TaskName, sql)
- // 源数据库执行sql查询结果
- columns, res, err := srcConn.Query(sql)
+ sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s", task.DataSql, updSql, orderSql)
+
+ err = app.doDataSync(sql, task)
if err != nil {
- app.endRunning(task, entity.DataSyncTaskStateFail, fmt.Sprintf("查询源数据库失败:%s", err.Error()), sql, 0)
- return
+ app.endRunning(task, entity.DataSyncTaskStateFail, fmt.Sprintf("执行失败: %s", err.Error()), sql, 0)
}
- if len(res) == 0 {
- app.endRunning(task, entity.DataSyncTaskStateSuccess, fmt.Sprintf("执行成功，新数据：%d 条", total), sql, 0)
- return
- }
- // 每次分页查询成功后，记录一些数据
- resSize = len(res)
- total += resSize
- page++
- index := 0
+ }()
- // task.FieldMap为json数组字符串 [{"src":"id","target":"id"}]，转为map
- var fieldMap []map[string]string
- err = json.Unmarshal([]byte(task.FieldMap), &fieldMap)
- if err != nil {
- app.endRunning(task, entity.DataSyncTaskStateFail, "解析字段映射json出错", sql, resSize)
- return
- }
+ return nil
+}
- // 遍历columns 取task.UpdField的字段类型
- updFieldType := dbm.DataTypeString
- for _, column := range columns {
- if column.Name == task.UpdField {
- updFieldType = srcDialect.GetDataType(column.Type)
- break
- }
- }
-
- var data = make([]map[string]any, 0)
-
- // 遍历res，组装插入sql
- for _, record := range res {
- index++
- // 获取查询结果最后一条数据的UpdField字段值
- if index == resSize {
- updFieldVal = fmt.Sprintf("%v", record[task.UpdField])
- updFieldVal = srcDialect.FormatStrData(updFieldVal, updFieldType)
- }
- var rowData = make(map[string]any)
- // 遍历字段映射, target字段的值为src字段取值
- for _, item := range fieldMap {
- srcField := item["src"]
- targetField := item["target"]
- // target字段的值为src字段取值
- rowData[targetField] = record[srcField]
- }
-
- data = append(data, rowData)
- }
-
- // 获取目标库字段数组
- targetWrapColumns := make([]string, 0)
- // 获取源库字段数组
- srcColumns := make([]string, 0)
- for _, item := range fieldMap {
- targetField := item["target"]
- srcField := item["target"]
- targetWrapColumns = append(targetWrapColumns, targetDialect.WrapName(targetField))
- srcColumns = append(srcColumns, srcField)
- }
-
- // 从目标库数据中取出源库字段对应的值
- values := make([][]any, 0)
- for _, record := range data {
- rawValue := make([]any, 0)
- for _, column := range srcColumns {
- rawValue = append(rawValue, record[column])
- }
- values = append(values, rawValue)
- }
-
- // 生成占位符字符串：如：(?,?)
- // 重复字符串并用逗号连接
- repeated := strings.Repeat("?,", len(targetWrapColumns))
- // 去除最后一个逗号，占位符由括号包裹
- placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
-
- // 目标数据库执行sql批量插入
- err = targetDialect.SaveBatch(targetConn, task.TargetTableName, strings.Join(targetWrapColumns, ","), placeholder, values)
- if err != nil {
- // 保存执行成功日志
- logx.Errorf("保存记录失败：%s", err.Error())
- app.endRunning(task, entity.DataSyncTaskStateFail, err.Error(), sql, resSize)
- return
- }
-
- // 保存运行时日志
- logx.Infof("同步任务：[%s],保存记录成功：[%d]条", task.TaskName, total)
- app.saveLog(task.Id, entity.DataSyncTaskStateSuccess, fmt.Sprintf("分页执行成功，新数据：%d 条", total), sql, total)
-
- // 运行过程中，判断状态是否为已关闭，是则结束运行，否则继续运行
- taskParam, _ := app.GetById(new(entity.DataSyncTask), id)
- if taskParam.RunningState == entity.DataSyncTaskRunStateStop {
- app.endRunning(task, entity.DataSyncTaskStateFail, "手动停止任务", sql, resSize)
- return
- }
-
- // 记录一次数据状态
- taskParam = new(entity.DataSyncTask)
- taskParam.Id = task.Id
- taskParam.UpdFieldVal = updFieldVal
- taskParam.RecentState = entity.DataSyncTaskStateSuccess
- taskParam.RunningState = entity.DataSyncTaskRunStateRunning
- _ = app.UpdateById(context.Background(), taskParam)
+func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) error {
+ // 获取源数据库连接
+ srcConn, err := GetDbApp().GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
+ if err != nil {
+ return errorx.NewBiz("连接源数据库失败: %s", err.Error())
}
- logx.Infof("同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
+ // 获取目标数据库连接
+ targetConn, err := GetDbApp().GetDbConn(uint64(task.TargetDbId), task.TargetDbName)
+ if err != nil {
+ return errorx.NewBiz("连接目标数据库失败: %s", err.Error())
+ }
+ targetDbTx, err := targetConn.Begin()
+ if err != nil {
+ return errorx.NewBiz("开启目标数据库事务失败: %s", err.Error())
+ }
+ defer func() {
+ if r := recover(); r != nil {
+ targetDbTx.Rollback()
+ err = fmt.Errorf("%v", r)
+ }
+ }()
+ srcDialect := srcConn.GetDialect()
// 记录更新字段最新值
- task.UpdFieldVal = updFieldVal
+ targetDialect := targetConn.GetDialect()
+
+ // task.FieldMap为json数组字符串 [{"src":"id","target":"id"}]，转为map
+ var fieldMap []map[string]string
+ err = json.Unmarshal([]byte(task.FieldMap), &fieldMap)
+ if err != nil {
+ return errorx.NewBiz("解析字段映射json出错: %s", err.Error())
+ }
+ var updFieldType dbm.DataType
+
+ // 记录本次同步数据总数
+ total := 0
+ batchSize := task.PageSize
+ result := make([]map[string]any, 0)
+ var queryColumns []*dbm.QueryColumn
+
+ err = srcConn.WalkQueryRows(context.Background(), sql, func(row map[string]any, columns []*dbm.QueryColumn) error {
+ if len(queryColumns) == 0 {
+ queryColumns = columns
+
+ // 遍历columns 取task.UpdField的字段类型
+ updFieldType = dbm.DataTypeString
+ for _, column := range columns {
+ if column.Name == task.UpdField {
+ updFieldType = srcDialect.GetDataType(column.Type)
+ break
+ }
+ }
+ }
+
+ total++
+ result = append(result, row)
+ if total%batchSize == 0 {
+ if err := app.srcData2TargetDb(result, fieldMap, updFieldType, task, srcDialect, targetDialect, targetDbTx); err != nil {
+ return err
+ }
+ result = result[:0]
+ }
+
+ return nil
+ })
+
+ if err != nil {
+ targetDbTx.Rollback()
+ return err
+ }
+
+ // 处理剩余的数据
+ if len(result) > 0 {
+ if err := app.srcData2TargetDb(result, fieldMap, updFieldType, task, srcDialect, targetDialect, targetDbTx); err != nil {
+ targetDbTx.Rollback()
+ return err
+ }
+ }
+
+ if err := targetDbTx.Commit(); err != nil {
+ return errorx.NewBiz("数据同步-目标数据库事务提交失败: %s", err.Error())
+ }
+ logx.Infof("同步任务：[%s]，执行完毕，保存记录成功：[%d]条", task.TaskName, total)
// 保存执行成功日志
app.endRunning(task, entity.DataSyncTaskStateSuccess, fmt.Sprintf("本次任务执行成功，新数据：%d 条", total), "", total)
+ return nil
+}
+func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, updFieldType dbm.DataType, task *entity.DataSyncTask, srcDialect dbm.DbDialect, targetDialect dbm.DbDialect, targetDbTx *sql.Tx) error {
+ var data = make([]map[string]any, 0)
+
+ // 遍历res，组装插入sql
+ for _, record := range srcRes {
+ var rowData = make(map[string]any)
+ // 遍历字段映射, target字段的值为src字段取值
+ for _, item := range fieldMap {
+ srcField := item["src"]
+ targetField := item["target"]
+ // target字段的值为src字段取值
+ rowData[targetField] = record[srcField]
+ }
+
+ data = append(data, rowData)
+ }
+
+ updFieldVal := fmt.Sprintf("%v", srcRes[len(srcRes)-1][task.UpdField])
+ updFieldVal = srcDialect.FormatStrData(updFieldVal, updFieldType)
+ task.UpdFieldVal = updFieldVal
+
+ // 获取目标库字段数组
+ targetWrapColumns := make([]string, 0)
+ // 获取源库字段数组
+ srcColumns := make([]string, 0)
+ for _, item := range fieldMap {
+ targetField := item["target"]
+ srcField := item["target"]
+ targetWrapColumns = append(targetWrapColumns, targetDialect.WrapName(targetField))
+ srcColumns = append(srcColumns, srcField)
+ }
+
+ // 从目标库数据中取出源库字段对应的值
+ values := make([][]any, 0)
+ for _, record := range data {
+ rawValue := make([]any, 0)
+ for _, column := range srcColumns {
+ rawValue = append(rawValue, record[column])
+ }
+ values = append(values, rawValue)
+ }
+
+ // 目标数据库执行sql批量插入
+ _, err := targetDialect.BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values)
+ if err != nil {
+ return err
+ }
+
+ // 运行过程中，判断状态是否为已关闭，是则结束运行，否则继续运行
+ taskParam, _ := app.GetById(new(entity.DataSyncTask), task.Id)
+ if taskParam.RunningState == entity.DataSyncTaskRunStateStop {
+ return errorx.NewBiz("该任务已被手动终止")
+ }
+
+ return nil
}
func (app *dataSyncAppImpl) endRunning(taskEntity *entity.DataSyncTask, state int8, msg string, sql string, resNum int) {
-
logx.Info(msg)
task := new(entity.DataSyncTask)
task.Id = taskEntity.Id
task.RecentState = state
- task.UpdFieldVal = taskEntity.UpdFieldVal
+ if state == entity.DataSyncTaskStateSuccess {
+ task.UpdFieldVal = taskEntity.UpdFieldVal
+ }
task.RunningState = entity.DataSyncTaskRunStateReady
// 运行失败之后设置任务状态为禁用
//if state == entity.DataSyncTaskStateFail {
@@ -315,7 +322,6 @@ func (app *dataSyncAppImpl) endRunning(taskEntity *entity.DataSyncTask, state in
_ = app.UpdateById(context.Background(), task)
// 保存执行日志
app.saveLog(taskEntity.Id, state, msg, sql, resNum)
-
}
func (app *dataSyncAppImpl) saveLog(taskId uint64, state int8, msg string, sql string, resNum int) {
diff --git a/server/internal/db/dbm/conn.go b/server/internal/db/dbm/conn.go
index f56a3824..0b95cb41 100644
--- a/server/internal/db/dbm/conn.go
+++ b/server/internal/db/dbm/conn.go
@@ -12,6 +12,9 @@ import (
"strings"
)
+// 游标遍历查询结果集处理函数
+type WalkQueryRowsFunc func(row map[string]any, columns []*QueryColumn) error
+
// db实例连接信息
type DbConn struct {
Id string
@@ -36,13 +39,21 @@ func (d *DbConn) Query(querySql string, args ...any) ([]*QueryColumn, []map[stri
// 依次返回 列信息数组(顺序)，结果map，错误
func (d *DbConn) QueryContext(ctx context.Context, querySql string, args ...any) ([]*QueryColumn, []map[string]any, error) {
result := make([]map[string]any, 0, 16)
- columns, err := walkTableRecord(ctx, d.db, querySql, func(record map[string]any, columns []*QueryColumn) {
- result = append(result, record)
+ var queryColumns []*QueryColumn
+
+ err := d.WalkQueryRows(ctx, querySql, func(row map[string]any, columns []*QueryColumn) error {
+ if len(queryColumns) == 0 {
+ queryColumns = columns
+ }
+ result = append(result, row)
+ return nil
}, args...)
+
if err != nil {
return nil, nil, wrapSqlError(err)
}
- return columns, result, nil
+
+ return queryColumns, result, nil
}
// 将查询结果映射至struct，可具体参考sqlx库
@@ -61,10 +72,9 @@ func (d *DbConn) Query2Struct(execSql string, dest any) error {
return scanAll(rows, dest, false)
}
-// WalkTableRecord 遍历表记录
-func (d *DbConn) WalkTableRecord(ctx context.Context, selectSql string, walk func(record map[string]any, columns []*QueryColumn)) error {
- _, err := walkTableRecord(ctx, d.db, selectSql, walk)
- return err
+// 游标方式遍历查询结果集, walkFn返回error不为nil, 则跳出遍历
+func (d *DbConn) WalkQueryRows(ctx context.Context, querySql string, walkFn WalkQueryRowsFunc, args ...any) error {
+ return walkQueryRows(ctx, d.db, querySql, walkFn, args...)
}
// 执行 update, insert, delete，建表等sql
@@ -73,16 +83,44 @@ func (d *DbConn) Exec(sql string, args ...any) (int64, error) {
return d.ExecContext(context.Background(), sql, args...)
}
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不使用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExec(tx *sql.Tx, execSql string, args ...any) (int64, error) {
+ return d.TxExecContext(context.Background(), tx, execSql, args...)
+}
+
// 执行 update, insert, delete，建表等sql
// 返回影响条数和错误
-func (d *DbConn) ExecContext(ctx context.Context, sql string, args ...any) (int64, error) {
- res, err := d.db.ExecContext(ctx, sql, args...)
+func (d *DbConn) ExecContext(ctx context.Context, execSql string, args ...any) (int64, error) {
+ res, err := d.db.ExecContext(ctx, execSql, args...)
if err != nil {
return 0, wrapSqlError(err)
}
return res.RowsAffected()
}
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不适用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExecContext(ctx context.Context, tx *sql.Tx, execSql string, args ...any) (int64, error) {
+ var res sql.Result
+ var err error
+ if tx != nil {
+ res, err = tx.ExecContext(ctx, execSql, args...)
+ } else {
+ res, err = d.db.ExecContext(ctx, execSql, args...)
+ }
+
+ if err != nil {
+ return 0, wrapSqlError(err)
+ }
+ return res.RowsAffected()
+}
+
+// 开启事务
+func (d *DbConn) Begin() (*sql.Tx, error) {
+ return d.db.Begin()
+}
+
// 获取数据库元信息实现接口
func (d *DbConn) GetDialect() DbDialect {
switch d.Info.Type {
@@ -111,11 +149,12 @@ func (d *DbConn) Close() {
}
}
-func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk func(record map[string]any, columns []*QueryColumn), args ...any) ([]*QueryColumn, error) {
+// 游标方式遍历查询rows, walkFn error不为nil, 则跳出遍历
+func walkQueryRows(ctx context.Context, db *sql.DB, selectSql string, walkFn WalkQueryRowsFunc, args ...any) error {
rows, err := db.QueryContext(ctx, selectSql, args...)
if err != nil {
- return nil, err
+ return err
}
// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
// 后面的链接过来直接报错或拒绝，实际上也没有起效果
@@ -127,7 +166,7 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
colTypes, err := rows.ColumnTypes()
if err != nil {
- return nil, err
+ return err
}
lenCols := len(colTypes)
// 列名用于前端表头名称按照数据库与查询字段顺序显示
@@ -145,7 +184,7 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
for rows.Next() {
// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
if err := rows.Scan(scans...); err != nil {
- return nil, err
+ return err
}
// 每行数据
rowData := make(map[string]any, lenCols)
@@ -153,10 +192,13 @@ func walkTableRecord(ctx context.Context, db *sql.DB, selectSql string, walk fun
for i, v := range values {
rowData[colTypes[i].Name()] = valueConvert(v, colTypes[i])
}
- walk(rowData, cols)
+ if err = walkFn(rowData, cols); err != nil {
+ logx.Error("游标遍历查询结果集出错,退出遍历: %s", err.Error())
+ return err
+ }
}
- return cols, nil
+ return nil
}
// 将查询的值转为对应列类型的实际值，不全部转为字符串
diff --git a/server/internal/db/dbm/dialect.go b/server/internal/db/dbm/dialect.go
index d1bd5dad..108d0e03 100644
--- a/server/internal/db/dbm/dialect.go
+++ b/server/internal/db/dbm/dialect.go
@@ -1,6 +1,7 @@
package dbm
import (
+ "database/sql"
"embed"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/utils/collx"
@@ -71,12 +72,8 @@ type DbDialect interface {
// 获取建表ddl
GetTableDDL(tableName string) (string, error)
- // 获取指定表的数据-分页查询
- // @return columns: 列字段名；result: 结果集；error: 错误
- GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error)
-
// WalkTableRecord 遍历指定表的数据
- WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error
+ WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error
GetSchemas() ([]string, error)
@@ -86,11 +83,8 @@ type DbDialect interface {
// 封装名字，如，mysql: `table_name`, dm: "table_name"
WrapName(name string) string
- // 分页sql，如：mysql: limit 1 ,10, dm: limit 10 offset 1
- PageSql(pageNum int, pageSize int) string
-
// 批量保存数据
- SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error
+ BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error)
GetDataType(dbColumnType string) DataType
diff --git a/server/internal/db/dbm/dialect_dm.go b/server/internal/db/dbm/dialect_dm.go
index 88d679ec..6bf719f2 100644
--- a/server/internal/db/dbm/dialect_dm.go
+++ b/server/internal/db/dbm/dialect_dm.go
@@ -255,12 +255,8 @@ func (dd *DMDialect) GetTableDDL(tableName string) (string, error) {
return builder.String(), nil
}
-func (dd *DMDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
- return dd.dc.Query(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
-}
-
-func (dd *DMDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
- return dd.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
+func (dd *DMDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
+ return dd.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
}
// 获取DM当前连接的库可访问的schemaNames
@@ -286,10 +282,6 @@ func (pd *DMDialect) WrapName(name string) string {
return "\"" + name + "\""
}
-func (pd *DMDialect) PageSql(pageNum int, pageSize int) string {
- return fmt.Sprintf("LIMIT %d OFFSET %d", pageSize, (pageNum-1)*pageSize)
-}
-
func (pd *DMDialect) GetDataType(dbColumnType string) DataType {
if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
return DataTypeNumber
@@ -309,21 +301,29 @@ func (pd *DMDialect) GetDataType(dbColumnType string) DataType {
return DataTypeString
}
-func (pd *DMDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
// 执行批量insert sql
// insert into "table_name" ("column1", "column2", ...) values (value1, value2, ...)
- sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+ // 生成占位符字符串：如：(?,?)
+ // 重复字符串并用逗号连接
+ repeated := strings.Repeat("?,", len(columns))
+ // 去除最后一个逗号，占位符由括号包裹
+ placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
+ sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
+ effRows := 0
for _, value := range values {
// 达梦数据库只能一条条的执行insert
- _, err := conn.Exec(sqlTemp, value...)
+ er, err := pd.dc.TxExec(tx, sqlTemp, value...)
if err != nil {
logx.Errorf("执行sql失败：%s", err.Error())
- return err
+ return int64(effRows), err
}
+ effRows += int(er)
}
// 执行批量insert sql
- return nil
+ return int64(effRows), nil
}
func (pd *DMDialect) FormatStrData(dbColumnValue string, dataType DataType) string {
diff --git a/server/internal/db/dbm/dialect_mysql.go b/server/internal/db/dbm/dialect_mysql.go
index f9141c2a..6413e751 100644
--- a/server/internal/db/dbm/dialect_mysql.go
+++ b/server/internal/db/dbm/dialect_mysql.go
@@ -189,12 +189,8 @@ func (md *MysqlDialect) GetTableDDL(tableName string) (string, error) {
return res[0]["Create Table"].(string) + ";", nil
}
-func (md *MysqlDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
- return md.dc.Query(fmt.Sprintf("SELECT * FROM %s LIMIT %d, %d", tableName, (pageNum-1)*pageSize, pageSize))
-}
-
-func (md *MysqlDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
- return md.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
+func (md *MysqlDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
+ return md.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
}
func (md *MysqlDialect) GetSchemas() ([]string, error) {
@@ -210,10 +206,6 @@ func (pd *MysqlDialect) WrapName(name string) string {
return "`" + name + "`"
}
-func (pd *MysqlDialect) PageSql(pageNum int, pageSize int) string {
- return fmt.Sprintf("limit %d, %d", (pageNum-1)*pageSize, pageSize)
-}
-
func (pd *MysqlDialect) GetDataType(dbColumnType string) DataType {
if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
return DataTypeNumber
@@ -233,24 +225,29 @@ func (pd *MysqlDialect) GetDataType(dbColumnType string) DataType {
return DataTypeString
}
-func (pd *MysqlDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
+ // 生成占位符字符串：如：(?,?)
+ // 重复字符串并用逗号连接
+ repeated := strings.Repeat("?,", len(columns))
+ // 去除最后一个逗号，占位符由括号包裹
+ placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
// 执行批量insert sql，mysql支持批量insert语法
// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
// 重复占位符字符串n遍
- repeated := strings.Repeat(placeholder+",", len(values))
+ repeated = strings.Repeat(placeholder+",", len(values))
// 去除最后一个逗号
placeholder = strings.TrimSuffix(repeated, ",")
- sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+ sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
// 执行批量insert sql
// 把二维数组转为一维数组
var args []any
for _, v := range values {
args = append(args, v...)
}
- _, err := conn.Exec(sqlStr, args...)
- return err
+ return pd.dc.TxExec(tx, sqlStr, args...)
}
func (pd *MysqlDialect) FormatStrData(dbColumnValue string, dataType DataType) string {
diff --git a/server/internal/db/dbm/dialect_pgsql.go b/server/internal/db/dbm/dialect_pgsql.go
index 3f79334d..b4d3a1ca 100644
--- a/server/internal/db/dbm/dialect_pgsql.go
+++ b/server/internal/db/dbm/dialect_pgsql.go
@@ -257,12 +257,8 @@ func (pd *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
return res[0]["sql"].(string), nil
}
-func (pd *PgsqlDialect) GetTableRecord(tableName string, pageNum, pageSize int) ([]*QueryColumn, []map[string]any, error) {
- return pd.dc.Query(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
-}
-
-func (pd *PgsqlDialect) WalkTableRecord(tableName string, walk func(record map[string]any, columns []*QueryColumn)) error {
- return pd.dc.WalkTableRecord(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walk)
+func (pd *PgsqlDialect) WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error {
+ return pd.dc.WalkQueryRows(context.Background(), fmt.Sprintf("SELECT * FROM %s", tableName), walkFn)
}
// 获取pgsql当前连接的库可访问的schemaNames
@@ -288,10 +284,6 @@ func (pd *PgsqlDialect) WrapName(name string) string {
return name
}
-func (pd *PgsqlDialect) PageSql(pageNum int, pageSize int) string {
- return fmt.Sprintf("LIMIT %d OFFSET %d", pageSize, (pageNum-1)*pageSize)
-}
-
func (pd *PgsqlDialect) GetDataType(dbColumnType string) DataType {
if regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`).MatchString(dbColumnType) {
return DataTypeNumber
@@ -311,24 +303,33 @@ func (pd *PgsqlDialect) GetDataType(dbColumnType string) DataType {
return DataTypeString
}
-func (pd *PgsqlDialect) SaveBatch(conn *DbConn, tableName string, columns string, placeholder string, values [][]any) error {
+func (pd *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
// 执行批量insert sql，跟mysql一样 pg或高斯支持批量insert语法
// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
+ // 生成占位符字符串：如：(?,?)
+ // 重复字符串并用逗号连接
+ repeated := strings.Repeat("?,", len(columns))
+ // 去除最后一个逗号，占位符由括号包裹
+ placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
+
+ // 执行批量insert sql，mysql支持批量insert语法
+ // insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
+
// 重复占位符字符串n遍
- repeated := strings.Repeat(placeholder+",", len(values))
+ repeated = strings.Repeat(placeholder+",", len(values))
// 去除最后一个逗号
placeholder = strings.TrimSuffix(repeated, ",")
- sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), columns, placeholder)
+ sqlStr := fmt.Sprintf("insert into %s (%s) values %s", pd.WrapName(tableName), strings.Join(columns, ","), placeholder)
// 执行批量insert sql
// 把二维数组转为一维数组
var args []any
for _, v := range values {
args = append(args, v...)
}
- _, err := conn.Exec(sqlStr, args...)
- return err
+
+ return pd.dc.TxExec(tx, sqlStr, args...)
}
func (pd *PgsqlDialect) FormatStrData(dbColumnValue string, dataType DataType) string {