TRKJ/mayfly-go
2
0
Fork 0
mirror of https://gitee.com/dromara/mayfly-go synced 2025-12-30 19:36:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

!106 feat：数据同步支持唯一键冲突策略

Browse Source 
* refactor：sql同步
* fix: 表格右键导出菜单换行符修复
* feat：数据同步支持唯一键冲突策略
This commit is contained in:
zongyangleo
2024-03-01 04:03:03 +00:00
committed by Coder慌
parent f93231da61
commit 76475e807e
38 changed files with 1117 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
server/internal/db/dbm/oracle/dialect.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
@@ -252,13 +253,7 @@ func (od *OracleDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", od.dc.Info.Type)
}
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
//INSERT ALL
//INTO my_table(field_1,field_2) VALUES (value_1,value_2)
//INTO my_table(field_1,field_2) VALUES (value_3,value_4)
//INTO my_table(field_1,field_2) VALUES (value_5,value_6)
//SELECT 1 FROM DUAL;
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
if len(values) <= 0 {
return 0, nil
}
@@ -269,27 +264,122 @@ func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
args = append(args, v...)
}
// 拼接oracle批量插入语句
sqlArr := make([]string, 0)
sqlArr = append(sqlArr, "INSERT ALL")
if duplicateStrategy == dbi.DuplicateStrategyNone || duplicateStrategy == 0 || duplicateStrategy == dbi.DuplicateStrategyIgnore {
return od.batchInsertSimple(od.dc.Info.Type, tableName, columns, values, duplicateStrategy, tx)
} else {
return od.batchInsertMergeSql(od.dc.Info.Type, tableName, columns, values, args, tx)
}
}
// 简单批量插入sql，无需判断键冲突策略
func (od *OracleDialect) batchInsertSimple(dbType dbi.DbType, tableName string, columns []string, values [][]any, duplicateStrategy int, tx *sql.Tx) (int64, error) {
// 忽略键冲突策略
ignore := ""
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
// 查出唯一索引涉及的字段
indexs, _ := od.GetTableIndex(tableName)
if indexs != nil {
arr := make([]string, 0)
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
for _, col := range cols {
if !collx.ArrayContains(arr, col) {
arr = append(arr, col)
}
}
}
}
ignore = fmt.Sprintf("/*+ IGNORE_ROW_ON_DUPKEY_INDEX(%s(%s)) */", tableName, strings.Join(arr, ","))
}
}
effRows := 0
for _, value := range values {
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
var placeholder []string
for i := 0; i < len(value); i++ {
placeholder = append(placeholder, fmt.Sprintf(":%d", i+1))
}
sqlTemp := fmt.Sprintf("INSERT %s INTO %s (%s) VALUES (%s)", ignore, dbType.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ","))
// oracle数据库为了兼容ignore主键冲突，只能一条条的执行insert
res, err := od.dc.TxExec(tx, sqlTemp, value...)
if err != nil {
logx.Errorf("执行sql失败：%s, sql: [ %s ]", err.Error(), sqlTemp)
}
effRows += int(res)
}
return int64(effRows), nil
}
func (od *OracleDialect) batchInsertMergeSql(dbType dbi.DbType, tableName string, columns []string, values [][]any, args []any, tx *sql.Tx) (int64, error) {
// 查询主键字段
uniqueCols := make([]string, 0)
caseSqls := make([]string, 0)
// 查询唯一索引涉及到的字段，并组装到match条件内
indexs, _ := od.GetTableIndex(tableName)
if indexs != nil {
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
tmp := make([]string, 0)
for _, col := range cols {
if !collx.ArrayContains(uniqueCols, col) {
uniqueCols = append(uniqueCols, col)
}
tmp = append(tmp, fmt.Sprintf(" T1.%s = T2.%s ", dbType.QuoteIdentifier(col), dbType.QuoteIdentifier(col)))
}
caseSqls = append(caseSqls, fmt.Sprintf("( %s )", strings.Join(tmp, " AND ")))
}
}
}
// 如果caseSqls为空，则说明没有唯一键，直接使用简单批量插入
if len(caseSqls) == 0 {
return od.batchInsertSimple(dbType, tableName, columns, values, dbi.DuplicateStrategyNone, tx)
}
// 重复数据处理策略
insertVals := make([]string, 0)
upds := make([]string, 0)
insertCols := make([]string, 0)
for _, column := range columns {
if !collx.ArrayContains(uniqueCols, dbType.RemoveQuote(column)) {
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
}
insertCols = append(insertCols, fmt.Sprintf("T1.%s", column))
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
}
// 生成源数据占位sql
t2s := make([]string, 0)
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
for i := 0; i < len(args); i += len(columns) {
var placeholder []string
for j := 0; j < len(columns); j++ {
placeholder = append(placeholder, fmt.Sprintf(":%d", i+j+1))
col := columns[j]
placeholder = append(placeholder, fmt.Sprintf(":%d %s", i+j+1, col))
}
sqlArr = append(sqlArr, fmt.Sprintf("INTO %s (%s) VALUES (%s)", od.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ",")))
t2s = append(t2s, fmt.Sprintf("SELECT %s FROM dual", strings.Join(placeholder, ", ")))
}
sqlArr = append(sqlArr, "SELECT 1 FROM DUAL")
t2 := strings.Join(t2s, " UNION ALL ")
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON (" + strings.Join(caseSqls, " OR ") + ") "
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ") "
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
// 执行批量insert sql
res, err := od.dc.TxExec(tx, strings.Join(sqlArr, " "), args...)
res, err := od.dc.TxExec(tx, sqlTemp, args...)
if err != nil {
logx.Errorf("执行sql失败：%s, sql: [ %s ]", err.Error(), sqlTemp)
}
return res, err
}
func (od *OracleDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
var (
@@ -297,6 +387,8 @@ var (
numberTypeRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeTypeRegexp = regexp.MustCompile(`(?i)date|timestamp`)
converter = new(DataConverter)
)
type DataConverter struct {