!106 feat:数据同步支持唯一键冲突策略

* refactor:sql同步
* fix: 表格右键导出菜单换行符修复
* feat:数据同步支持唯一键冲突策略
This commit is contained in:
zongyangleo
2024-03-01 04:03:03 +00:00
committed by Coder慌
parent f93231da61
commit 76475e807e
38 changed files with 1117 additions and 257 deletions

View File

@@ -7,24 +7,22 @@ export function exportCsv(filename: string, columns: string[], datas: []) {
for (let column of columns) {
let val: any = data[column];
if (val == null || val == undefined) {
dataValueArr.push('');
continue;
}
val = '';
} else if (val && typeof val == 'string') {
// 替换换行符
val = val.replace(/[\r\n]/g, '\\n');
if (typeof val == 'string' && val) {
// csv格式如果有逗号整体用双引号括起来如果里面还有双引号就替换成两个双引号这样导出来的格式就不会有问题了
if (val.indexOf(',') != -1) {
// 如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if (val.indexOf('"') != -1) {
val = val.replace(/\"/g, '""');
val = val.replace(/"/g, '""');
}
// 再将逗号转义
val = `"${val}"`;
}
dataValueArr.push(val + '\t');
} else {
dataValueArr.push(val + '\t');
}
dataValueArr.push(String(val));
}
cvsData.push(dataValueArr);
}

View File

@@ -95,6 +95,7 @@ import { ElMessage } from 'element-plus';
import TagTreeSelect from '../component/TagTreeSelect.vue';
import type { CheckboxValueType } from 'element-plus';
import ProcdefSelectFormItem from '@/views/flow/components/ProcdefSelectFormItem.vue';
import { DbType } from '@/views/ops/db/dialect';
const props = defineProps({
visible: {
@@ -196,7 +197,14 @@ const changeInstance = () => {
const getAllDatabase = async () => {
if (state.form.instanceId > 0) {
state.allDatabases = await dbApi.getAllDatabase.request({ instanceId: state.form.instanceId });
let dbs = await dbApi.getAllDatabase.request({ instanceId: state.form.instanceId });
state.allDatabases = dbs;
// 如果是oracle且没查出数据库列表则取实例sid
let instance = state.instances.find((item: any) => item.id === state.form.instanceId);
if (instance && instance.type === DbType.oracle && dbs.length === 0) {
state.allDatabases = [instance.sid];
}
}
};

View File

@@ -61,7 +61,7 @@
</template>
<script lang="ts" setup>
import { ref, toRefs, reactive, onMounted, defineAsyncComponent, Ref } from 'vue';
import { defineAsyncComponent, onMounted, reactive, ref, Ref, toRefs } from 'vue';
import { ElMessage, ElMessageBox } from 'element-plus';
import { dbApi } from './api';
import { dateFormat } from '@/common/utils/date';

View File

@@ -685,8 +685,9 @@ const onDeleteTable = async (data: any) => {
cancelButtonText: '取消',
type: 'warning',
});
// 执行sql
dbApi.sqlExec.request({ id, db, sql: `drop table ${tableName}` }).then(() => {
dbApi.sqlExec.request({ id, db, sql: `drop table ${getDbDialect(state.nowDbInst.type).quoteIdentifier(tableName)}` }).then(() => {
if (flowProcdefKey) {
ElMessage.success('工单提交成功');
return;

View File

@@ -16,7 +16,7 @@
<el-row>
<el-col :span="11">
<el-form-item prop="taskName" label="任务名" required>
<el-input v-model.trim="form.taskName" placeholder="请输入数据库别名" auto-complete="off" />
<el-input v-model.trim="form.taskName" placeholder="请输入同步任务名" auto-complete="off" />
</el-form-item>
</el-col>
@@ -89,9 +89,11 @@
</el-col>
<el-col :span="8">
<el-tooltip content="查询数据源的时候会带上这个字段当前最大值支持带别名t.create_time" placement="top">
<el-form-item prop="updField" label="更新字段" required>
<el-input v-model.trim="form.updField" placeholder="查询数据源的时候会带上这个字段当前最大值" auto-complete="off" />
</el-form-item>
</el-tooltip>
</el-col>
<el-col :span="8">
@@ -125,10 +127,17 @@
<el-tab-pane label="sql预览" :name="sqlPreviewTab" :disabled="!baseFieldCompleted">
<el-form-item prop="fieldMap" label="查询sql">
<el-input type="textarea" v-model="state.previewDataSql" readonly :input-style="{ height: '190px' }" />
<el-input type="textarea" v-model="state.previewDataSql" readonly :input-style="{ height: '170px' }" />
</el-form-item>
<el-form-item prop="fieldMap" label="插入sql">
<el-input type="textarea" v-model="state.previewInsertSql" readonly :input-style="{ height: '190px' }" />
<el-input type="textarea" v-model="state.previewInsertSql" readonly :input-style="{ height: '170px' }" />
</el-form-item>
<el-form-item prop="isReplace" v-if="compatibleDuplicateStrategy(form.targetDbType!)" label="键冲突策略">
<el-select v-model="form.duplicateStrategy" @change="handleDuplicateStrategy" style="width: 100px">
<el-option label="无" :value="DuplicateStrategy.NONE" />
<el-option label="忽略" :value="DuplicateStrategy.IGNORE" />
<el-option label="替换" :value="DuplicateStrategy.REPLACE" />
</el-select>
</el-form-item>
</el-tab-pane>
</el-tabs>
@@ -185,7 +194,7 @@ import { ElMessage } from 'element-plus';
import DbSelectTree from '@/views/ops/db/component/DbSelectTree.vue';
import MonacoEditor from '@/components/monaco/MonacoEditor.vue';
import { DbInst, registerDbCompletionItemProvider } from '@/views/ops/db/db';
import { DbType, getDbDialect } from '@/views/ops/db/dialect';
import { compatibleDuplicateStrategy, DbType, DuplicateStrategy, getDbDialect } from '@/views/ops/db/dialect';
import CrontabInput from '@/components/crontab/CrontabInput.vue';
const props = defineProps({
@@ -246,6 +255,7 @@ type FormData = {
updFieldVal?: string;
fieldMap?: { src: string; target: string }[];
status?: 1 | 2;
duplicateStrategy?: -1 | 1 | 2;
};
const basicFormData = {
@@ -257,6 +267,7 @@ const basicFormData = {
updFieldVal: '0',
fieldMap: [{ src: 'a', target: 'b' }],
status: 1,
duplicateStrategy: -1,
} as FormData;
const state = reactive({
@@ -271,6 +282,7 @@ const state = reactive({
previewRes: {} as any,
previewDataSql: '',
previewInsertSql: '',
previewFieldArr: [] as string[],
});
const { tabActiveName, form, submitForm } = toRefs(state);
@@ -295,6 +307,9 @@ watch(dialogVisible, async (newValue: boolean) => {
let data = await dbApi.getDatasyncTask.request({ taskId: propsData?.id });
state.form = data;
if (!state.form.duplicateStrategy) {
state.form.duplicateStrategy = -1;
}
try {
state.form.fieldMap = JSON.parse(data.fieldMap);
} catch (e) {
@@ -343,13 +358,12 @@ watch(tabActiveName, async (newValue: string) => {
await handleGetTargetFields();
break;
case sqlPreviewTab:
let srcDbDialect = getDbDialect(state.srcDbInst.type);
let targetDbDialect = getDbDialect(state.targetDbInst.type);
let updField = state.form.updField!;
let updField = srcDbDialect.quoteIdentifier(state.form.updField!);
state.previewDataSql = `SELECT * FROM (\n ${state.form.dataSql?.trim() || '请输入数据sql'} \n ) t \n where ${updField} > '${
state.form.updFieldVal || ''
}'`;
// 判断sql是否以where .*结尾
let hasCondition = /where/i.test(state.form.dataSql!);
state.previewDataSql = `${state.form.dataSql?.trim() || '请输入数据sql'} \n ${hasCondition ? 'and' : 'where'} ${updField} > '${state.form.updFieldVal || ''}'`;
// 检查字段映射中是否存在重复的目标字段
let fields = new Set();
@@ -365,17 +379,19 @@ watch(tabActiveName, async (newValue: string) => {
}
let fieldArr = state.form.fieldMap?.map((a: any) => targetDbDialect.quoteIdentifier(a.target)) || [];
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
state.previewInsertSql = ` insert into ${targetDbDialect.quoteIdentifier(state.form.targetTableName!)}(${fieldArr.join(
','
)}) values (${placeholder});`;
state.previewFieldArr = fieldArr;
refreshPreviewInsertSql();
break;
default:
break;
}
});
const refreshPreviewInsertSql = () => {
let targetDbDialect = getDbDialect(state.targetDbInst.type);
state.previewInsertSql = targetDbDialect.getBatchInsertPreviewSql(state.form.targetTableName!, state.previewFieldArr, state.form.duplicateStrategy!);
};
const onSelectSrcDb = async (params: any) => {
// 初始化数据源
params.databases = params.dbs; // 数据源里需要这个值
@@ -422,16 +438,24 @@ const handleGetSrcFields = async () => {
}
// 执行sql
let sql: string;
if (state.form.srcDbType === DbType.mssql) {
// mssql的分页语法不一样
let top1 = `select top 1`;
sql = `${top1} * from (${state.form.dataSql}) a`;
} else if (state.form.srcDbType === DbType.oracle) {
// oracle的分页关键字不一样
let limit = ' limit 1';
if (state.form.srcDbType === DbType.oracle) {
limit = ' where rownum <= 1';
let hasCondition = /where/i.test(state.form.dataSql!);
sql = `${state.form.dataSql} ${hasCondition ? 'and' : 'where'} rownum <= 1`;
} else {
sql = `${state.form.dataSql} limit 1`;
}
const res = await dbApi.sqlExec.request({
id: state.form.srcDbId,
db: state.form.srcDbName,
sql: `select * from (${state.form.dataSql}) t ${limit}`,
sql,
});
if (!res.columns) {
@@ -505,6 +529,11 @@ const btnOk = async () => {
const cancel = () => {
dialogVisible.value = false;
emit('cancel');
state.form = basicFormData;
};
const handleDuplicateStrategy = () => {
refreshPreviewInsertSql();
};
</script>
<style lang="scss">

View File

@@ -742,6 +742,7 @@ const onExitEditMode = (rowData: any, column: any, rowIndex = 0) => {
};
const submitUpdateFields = async () => {
debugger;
const dbInst = getNowDbInst();
if (cellUpdateMap.size == 0) {
return;

View File

@@ -37,6 +37,7 @@ import { ref, watch, onMounted } from 'vue';
import ColumnFormItem from './ColumnFormItem.vue';
import { DbInst } from '../../db';
import { ElMessage } from 'element-plus';
import { getDbDialect } from '@/views/ops/db/dialect';
export interface ColumnFormItemProps {
dbInst: DbInst;

View File

@@ -524,7 +524,7 @@ const onConfirmCondition = () => {
}
const row = conditionDialog.columnRow as any;
condition += `${row.columnName} ${conditionDialog.condition} `;
state.condition = condition + DbInst.wrapColumnValue(row.columnType, conditionDialog.value);
state.condition = condition + state.dbDialect.wrapValue(row.columnType, conditionDialog.value!);
onCancelCondition();
condInputRef.value.focus();
};

View File

@@ -30,7 +30,7 @@
<el-select v-else-if="item.prop === 'type'" filterable size="small" v-model="scope.row.type">
<el-option
v-for="pgsqlType in state.columnTypeList"
v-for="pgsqlType in getDbDialect(dbType).getInfo().columnTypes"
:key="pgsqlType.dataType"
:value="pgsqlType.udtName"
:label="pgsqlType.dataType"
@@ -172,7 +172,6 @@ const state = reactive({
dialogVisible: false,
btnloading: false,
activeName: '1',
columnTypeList: dbDialect.getInfo().columnTypes,
tableData: {
fields: {
colNames: [
@@ -347,22 +346,25 @@ const submit = async () => {
* @param nowArr 修改后的对象数组
* @param key 标志对象唯一属性
*/
const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { del: any[]; add: any[]; upd: any[] } => {
const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { del: any[]; add: any[]; upd: any[]; changed: boolean } => {
let data = {
del: [] as object[], // 删除的数据
add: [] as object[], // 新增的数据
upd: [] as object[], // 修改的数据
changed: false,
};
// 旧数据为空
if (oldArr && Array.isArray(oldArr) && oldArr.length === 0 && nowArr && Array.isArray(nowArr) && nowArr.length > 0) {
data.add = nowArr;
data.changed = true;
return data;
}
// 新数据为空
if (nowArr && Array.isArray(nowArr) && nowArr.length === 0 && oldArr && Array.isArray(oldArr) && oldArr.length > 0) {
data.del = oldArr;
data.changed = true;
return data;
}
@@ -378,6 +380,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
oldName && (newMap[oldName] = a);
if (!oldMap.hasOwnProperty(k) && (!oldName || (oldName && !oldMap.hasOwnProperty(oldName)))) {
// 新增
data.changed = true;
data.add.push(a);
}
});
@@ -387,6 +390,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
let newData = newMap[k];
if (!newData) {
// 删除
data.changed = true;
data.del.push(a);
} else {
// 判断每个字段是否相等,否则为修改
@@ -394,6 +398,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
let oldV = a[f];
let newV = newData[f];
if (oldV?.toString() !== newV?.toString()) {
data.changed = true;
data.upd.push(newData);
break;
}
@@ -416,13 +421,17 @@ const genSql = () => {
} else {
// 修改列
let changeColData = filterChangedData(state.tableData.fields.oldFields, state.tableData.fields.res, 'name');
let colSql = dbDialect.getModifyColumnSql(data, data.tableName, changeColData);
let colSql = changeColData.changed ? dbDialect.getModifyColumnSql(data, data.tableName, changeColData) : '';
// 修改索引
let changeIdxData = filterChangedData(state.tableData.indexs.oldIndexs, state.tableData.indexs.res, 'indexName');
let idxSql = dbDialect.getModifyIndexSql(data, data.tableName, changeIdxData);
let idxSql = changeColData.changed ? dbDialect.getModifyIndexSql(data, data.tableName, changeIdxData) : '';
// 修改表名
return colSql + ';' + idxSql;
let sqlArr = [];
colSql && sqlArr.push(colSql);
idxSql && sqlArr.push(idxSql);
return sqlArr.join(';');
}
};
@@ -432,7 +441,9 @@ const reset = () => {
state.tableData.tableName = '';
state.tableData.tableComment = '';
state.tableData.fields.res = [];
state.tableData.fields.oldFields = [];
state.tableData.indexs.res = [];
state.tableData.indexs.oldIndexs = [];
};
const indexChanges = (row: any) => {

View File

@@ -166,7 +166,7 @@ export class DbInst {
const db = this.getDb(dbName);
// 优先从 table map中获取
let columns = db.getColumns(table);
if (columns) {
if (columns && columns.length > 0) {
return columns;
}
console.log(`load columns -> dbName: ${dbName}, table: ${table}`);
@@ -258,7 +258,7 @@ export class DbInst {
// 获取指定表的默认查询sql
getDefaultSelectSql(db: string, table: string, condition: string, orderBy: string, pageNum: number, limit: number = DbInst.DefaultLimit) {
return getDbDialect(this.type).getDefaultSelectSql(db, table, condition, orderBy, pageNum, limit);
return this.getDialect().getDefaultSelectSql(db, table, condition, orderBy, pageNum, limit);
}
/**
@@ -266,12 +266,22 @@ export class DbInst {
* @param dbName 数据库名
* @param table 表名
* @param datas 要生成的数据
* @param dbDialect db方言
* @param skipNull 是否跳过空字段
*/
async genInsertSql(dbName: string, table: string, datas: any[], skipNull = false) {
if (!datas) {
return '';
}
let schema = '';
let arr = dbName.split('/');
if (arr.length == 1) {
schema = this.wrapName(dbName) + '.';
} else if (arr.length == 2) {
schema = this.wrapName(arr[1]) + '.';
}
let dbDialect = this.getDialect();
const columns = await this.loadColumns(dbName, table);
const sqls = [];
for (let data of datas) {
@@ -283,9 +293,9 @@ export class DbInst {
continue;
}
colNames.push(this.wrapName(colName));
values.push(DbInst.wrapValueByType(data[colName]));
values.push(dbDialect.wrapValue(column.dataType, data[colName]));
}
sqls.push(`INSERT INTO ${this.wrapName(table)} (${colNames.join(', ')}) VALUES(${values.join(', ')})`);
sqls.push(`INSERT INTO ${schema}${this.wrapName(table)} (${colNames.join(', ')}) VALUES(${values.join(', ')})`);
}
return sqls.join(';\n') + ';';
}
@@ -315,22 +325,23 @@ export class DbInst {
const v = columnValue[k];
// 更新字段列信息
const updateColumn = await this.loadTableColumn(dbName, table, k);
sql += ` ${this.wrapName(k)} = ${DbInst.wrapColumnValue(updateColumn.columnType, v, dialect)},`;
sql += ` ${this.wrapName(k)} = ${dialect.wrapValue(updateColumn.columnType, v)},`;
}
sql = sql.substring(0, sql.length - 1);
return (sql += ` WHERE ${this.wrapName(primaryKeyName)} = ${DbInst.wrapColumnValue(primaryKeyType, primaryKeyValue)} ;`);
return sql + ` WHERE ${this.wrapName(primaryKeyName)} = ${this.getDialect().wrapValue(primaryKeyType, primaryKeyValue)} ;`;
}
/**
* 生成根据主键删除的sql语句
* @param db 数据库名
* @param table 表名
* @param datas 要删除的记录
*/
async genDeleteByPrimaryKeysSql(db: string, table: string, datas: any[]) {
const primaryKey = await this.loadTableColumn(db, table);
const primaryKeyColumnName = primaryKey.columnName;
const ids = datas.map((d: any) => `${DbInst.wrapColumnValue(primaryKey.columnType, d[primaryKeyColumnName])}`).join(',');
const ids = datas.map((d: any) => `${this.getDialect().wrapValue(primaryKey.columnType, d[primaryKeyColumnName])}`).join(',');
return `DELETE FROM ${this.wrapName(table)} WHERE ${this.wrapName(primaryKeyColumnName)} IN (${ids})`;
}
@@ -351,9 +362,8 @@ export class DbInst {
/**
* 包裹数据库表名、字段名等,避免使用关键字为字段名或表名时报错
* @param table
* @param condition
* @returns
* @param name 表名、字段名、schema名
* @returns 包裹后的字符串
*/
wrapName = (name: string) => {
return this.getDialect().quoteIdentifier(name);
@@ -410,34 +420,6 @@ export class DbInst {
dbInstCache.clear();
}
/**
* 根据返回值包装值,若值为字符串类型则添加''
* @param val 值
* @returns 包装后的值
*/
static wrapValueByType = (val: any) => {
if (val == null) {
return 'NULL';
}
if (typeof val == 'number') {
return val;
}
return `'${val}'`;
};
/**
* 根据字段类型包装字段值,如为字符串等则添加‘’,数字类型则直接返回即可
*/
static wrapColumnValue(columnType: string, value: any, dbDialect?: DbDialect) {
if (this.isNumber(columnType)) {
return value;
}
if (!dbDialect) {
return `'${value}'`;
}
return dbDialect.wrapStrValue(columnType, value);
}
/**
* 判断字段类型是否为数字类型
* @param columnType 字段类型

View File

@@ -4,6 +4,7 @@ import {
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
@@ -682,8 +683,54 @@ class DMDialect implements DbDialect {
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (DbInst.isNumber(columnType)) {
return value;
}
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
// 替换
// MERGE INTO t_person T1
// USING (
// <foreach collection="list" item="item" index="index" separator="UNION ALL">
// SELECT
// #{item.id} id,
// #{item.mc} mc,
// #{item.sex} sex,
// #{item.age} age
// FROM dual
// </foreach>
// ) T2 ON (T1.id = T2.id )
// WHEN NOT MATCHED THEN INSERT(id, mc, sex,
// age) VALUES
// (T2.id, T2.mc, T2.sex, T2.age)
// WHEN MATCHED THEN UPDATE
// SET T1.mc = T2.mc,T1.sex = T2.sex,T1.age = T2.age
if (duplicateStrategy == DuplicateStrategy.REPLACE || duplicateStrategy == DuplicateStrategy.IGNORE) {
// 字段数组生成占位符sql
let phs = [];
let values = [];
for (let i = 0; i < fieldArr.length; i++) {
phs.push(`? ${fieldArr[i]}`);
values.push(`T2.${fieldArr[i]}`);
}
let placeholder = phs.join(',');
let sql = `MERGE INTO ${tableName} T1 USING
(
SELECT ${placeholder} FROM dual
) T2 ON (T1.id = T2.id)
WHEN NOT MATCHED THEN INSERT(${fieldArr.join(',')}) VALUES (${values.join(',')})
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
return sql;
} else {
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}), (${placeholder});`;
}
}
}

View File

@@ -1,5 +1,5 @@
import { PostgresqlDialect } from '@/views/ops/db/dialect/postgres_dialect';
import { DialectInfo } from '@/views/ops/db/dialect/index';
import { DialectInfo, DuplicateStrategy } from '@/views/ops/db/dialect/index';
let gsDialectInfo: DialectInfo;
export class GaussDialect extends PostgresqlDialect {
@@ -14,4 +14,17 @@ export class GaussDialect extends PostgresqlDialect {
gsDialectInfo.name = 'GaussDB';
return gsDialectInfo;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
// 构建占位符字符串 "($1, $2, $3 ...)"
let placeholder = fieldArr.map((_, i) => `$${i + 1}`).join(',');
let suffix = '';
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
suffix = '\nON DUPLICATE KEY UPDATE NOTHING';
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
suffix = '\n-- 执行前会删除唯一键涉及到的字段 \nON DUPLICATE KEY UPDATE ' + fieldArr.map((a) => `${a}=excluded.${a}`).join(',');
}
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}) ${suffix};`;
}
}

View File

@@ -146,6 +146,25 @@ export const compatibleMysql = (dbType: string): boolean => {
}
};
// 哪些数据库支持键冲突策略
export const compatibleDuplicateStrategy = (dbType: string): boolean => {
switch (dbType) {
case DbType.mysql:
case DbType.mariadb:
case DbType.postgresql:
case DbType.gauss:
case DbType.kingbaseEs:
case DbType.vastbase:
case DbType.dm:
case DbType.oracle:
case DbType.sqlite:
case DbType.mssql:
return true;
default:
return false;
}
};
export interface DbDialect {
/**
* 获取一些数据库默认信息
@@ -206,8 +225,22 @@ export interface DbDialect {
/** 通过数据库字段类型,返回基本数据类型 */
getDataType(columnType: string): DataType;
/** 包装字符串数据, 如oracle需要把date类型改为 to_date(str, 'yyyy-mm-dd hh24:mi:ss') */
wrapStrValue(columnType: string, value: string): string;
/** 包装字符串数据, 如oracle需要把date类型改为 to_date(str, 'yyyy-mm-dd hh24:mi:ss') mssql需要把中文字符串数据包装为 N'中文字符串' */
wrapValue(columnType: string, value: any): any;
/**
* 生成插入数据预览sql
* @param tableName 表名
* @param columns 列名
* @param duplicateStrategy 重复策略
*/
getBatchInsertPreviewSql(tableName: string, columns: string[], duplicateStrategy: DuplicateStrategy): string;
}
export enum DuplicateStrategy {
NONE = -1, // 无
IGNORE = 1, // 忽略
REPLACE = 2, // 覆盖
}
let mysqlDialect = new MysqlDialect();

View File

@@ -1,5 +1,15 @@
import { DbInst } from '../db';
import { commonCustomKeywords, DataType, DbDialect, DialectInfo, EditorCompletion, EditorCompletionItem, IndexDefinition, RowDefinition } from './index';
import {
commonCustomKeywords,
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
RowDefinition,
} from './index';
import { language as sqlLanguage } from 'monaco-editor/esm/vs/basic-languages/sql/sql.js';
export { MSSQL_TYPE_LIST, MssqlDialect };
@@ -134,7 +144,7 @@ class MssqlDialect implements DbDialect {
{ name: 'creator_id', type: 'bigint', length: '20', numScale: '', value: '', notNull: true, pri: false, auto_increment: false, remark: '创建人id' },
{
name: 'creator',
type: 'varchar',
type: 'nvarchar',
length: '100',
numScale: '',
value: '',
@@ -157,7 +167,7 @@ class MssqlDialect implements DbDialect {
{ name: 'updator_id', type: 'bigint', length: '20', numScale: '', value: '', notNull: true, pri: false, auto_increment: false, remark: '修改人id' },
{
name: 'updator',
type: 'varchar',
type: 'nvarchar',
length: '100',
numScale: '',
value: '',
@@ -263,7 +273,10 @@ class MssqlDialect implements DbDialect {
}
});
return sql.join(';') + ';' + indexComment.join(';');
let arr = [];
sql.length > 0 && arr.push(sql.join(';'));
indexComment.length > 0 && arr.push(indexComment.join(';'));
return arr.join(';');
}
getModifyColumnSql(tableData: any, tableName: string, changeData: { del: RowDefinition[]; add: RowDefinition[]; upd: RowDefinition[] }): string {
@@ -326,14 +339,15 @@ ELSE
});
}
return (
delSql +
(addArr.length > 0 ? addArr.join(';') + ';' : '') +
(renameArr.length > 0 ? renameArr.join(';') + ';' : '') +
(updArr.length > 0 ? updArr.join(';') + ';' : '') +
(changeCommentArr.length > 0 ? changeCommentArr.join(';') + ';' : '') +
(addCommentArr.length > 0 ? addCommentArr.join(';') + ';' : '')
);
let arr = [];
delSql && arr.push(delSql);
addArr.length > 0 && arr.push(addArr.join(';'));
renameArr.length > 0 && arr.push(renameArr.join(';'));
updArr.length > 0 && arr.push(updArr.join(';'));
changeCommentArr.length > 0 && arr.push(changeCommentArr.join(';'));
addCommentArr.length > 0 && arr.push(addCommentArr.join(';'));
return arr.join(';');
}
getModifyIndexSql(tableData: any, tableName: string, changeData: { del: any[]; add: any[]; upd: any[] }): string {
@@ -377,7 +391,12 @@ ELSE
let dropSql = dropArr.join(';');
let addSql = addArr.join(';');
let commentSql = commentArr.join(';');
return dropSql + ';' + addSql + ';' + commentSql + ';';
let arr = [];
dropSql && arr.push(dropSql);
addSql && arr.push(addSql);
commentSql && arr.push(commentSql);
return arr.join(';');
}
getDataType(columnType: string): DataType {
@@ -398,8 +417,46 @@ ELSE
}
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (this.getDataType(columnType) == DataType.Number) {
return value;
}
if (this.getDataType(columnType) == DataType.String) {
return `N'${value}'`;
}
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
let baseSql = `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder});`;
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
let on = `ALTER TABLE ${tableName} ADD CONSTRAINT uniqueRows UNIQUE (id) WITH (IGNORE_DUP_KEY = ON);`;
return on + '\n' + baseSql;
}
if (duplicateStrategy === DuplicateStrategy.REPLACE) {
// 字段数组生成占位符sql
let phs = [];
let values = [];
for (let i = 0; i < fieldArr.length; i++) {
phs.push(`? ${fieldArr[i]}`);
values.push(`T2.${fieldArr[i]}`);
}
let placeholder = phs.join(',');
let sql = `MERGE INTO ${tableName} T1 USING
(
SELECT ${placeholder}
) T2 ON (T1.id = T2.id)
WHEN NOT MATCHED THEN INSERT(${fieldArr.join(',')}) VALUES (${values.join(',')})
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
return sql;
}
return baseSql;
}
}

View File

@@ -1,5 +1,15 @@
import { DbInst } from '../db';
import { commonCustomKeywords, DataType, DbDialect, DialectInfo, EditorCompletion, EditorCompletionItem, IndexDefinition, RowDefinition } from './index';
import {
commonCustomKeywords,
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
RowDefinition,
} from './index';
import { language as mysqlLanguage } from 'monaco-editor/esm/vs/basic-languages/mysql/mysql.js';
export { MYSQL_TYPE_LIST, MysqlDialect };
@@ -227,7 +237,6 @@ class MysqlDialect implements DbDialect {
}
getModifyColumnSql(tableData: any, tableName: string, changeData: { del: RowDefinition[]; add: RowDefinition[]; upd: RowDefinition[] }): string {
let sql = `ALTER TABLE ${this.quoteIdentifier(tableData.db)}.${this.quoteIdentifier(tableName)}`;
let arr = [] as string[];
if (changeData.del.length > 0) {
changeData.del.forEach((a) => {
@@ -250,9 +259,14 @@ class MysqlDialect implements DbDialect {
});
}
if (arr.length > 0) {
let sql = `ALTER TABLE ${this.quoteIdentifier(tableData.db)}.${this.quoteIdentifier(tableName)}`;
return sql + arr.join(',') + ';';
}
return '';
}
getModifyIndexSql(tableData: any, tableName: string, changeData: { del: any[]; add: any[]; upd: any[] }): string {
// 搜集修改和删除的索引添加到drop index xx
// 收集新增和修改的索引添加到ADD xx
@@ -327,8 +341,27 @@ class MysqlDialect implements DbDialect {
}
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (DbInst.isNumber(columnType)) {
return value;
}
// 转义所有的换行符
value = value.replace(/[\r\n]/g, '\\n');
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: number): string {
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
let prefix = 'insert into';
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
prefix = 'insert ignore into';
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
prefix = 'replace into';
}
return `${prefix} ${this.quoteIdentifier(tableName)}(${fieldArr.join(',')}) values (${placeholder});`;
}
}

View File

@@ -4,6 +4,7 @@ import {
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
@@ -84,7 +85,7 @@ const replaceFunctions: EditorCompletionItem[] = [
{ label: 'CURRENT_TIMESTAMP', insertText: 'TIMESTAMP', description: '获取当前时间' },
// 转换函数
{ label: 'TO_CHAR', insertText: 'TO_CHAR(d|n[,fmt])', description: '把日期和数字转换为制定格式的字符串' },
{ label: 'TO_DATE', insertText: 'TO_DATE(X,[,fmt])', description: '把一个字符串以fmt格式转换成一个日期类型' },
{ label: 'TO_DATE', insertText: `TO_DATE(X, 'yyyy-MM-dd HH24:mi:ss')`, description: '把一个字符串以fmt格式转换成一个日期类型' },
{ label: 'TO_NUMBER', insertText: 'TO_NUMBER(X,[,fmt])', description: '把一个字符串以fmt格式转换为一个数字' },
{ label: 'TO_TIMESTAMP', insertText: 'TO_TIMESTAMP(X,[,fmt])', description: '把一个字符串以fmt格式转换为日期类型' },
// 其他
@@ -310,6 +311,7 @@ class OracleDialect implements DbDialect {
let createSql = '';
let tableCommentSql = '';
let columCommentSql = '';
let pris = [] as string[];
// 创建表结构
let fields: string[] = [];
@@ -319,9 +321,18 @@ class OracleDialect implements DbDialect {
if (item.remark) {
columCommentSql += ` COMMENT ON COLUMN ${dbTable}.${this.quoteIdentifier(item.name)} is '${item.remark}'; `;
}
// 主键
if (item.pri) {
pris.push(this.quoteIdentifier(item.name));
}
});
// 主键语句
let prisql = '';
if (pris.length > 0) {
prisql = ` CONSTRAINT "PK_${data.tableName}" PRIMARY KEY (${pris.join(',')});`;
}
// 建表
createSql = `CREATE TABLE ${dbTable} ( ${fields.join(',')} );`;
createSql = `CREATE TABLE ${dbTable} ( ${fields.join(',')} ) ${prisql ? ',' + prisql : ''};`;
// 表注释
if (data.tableComment) {
tableCommentSql = ` COMMENT ON TABLE ${dbTable} is '${data.tableComment}'; `;
@@ -376,7 +387,7 @@ class OracleDialect implements DbDialect {
}
modifyArr.push(` MODIFY (${this.genColumnBasicSql(a, false)})`);
if (a.pri) {
priArr.add(`${this.quoteIdentifier(a.name)}"`);
priArr.add(`${this.quoteIdentifier(a.name)}`);
}
});
}
@@ -412,8 +423,8 @@ class OracleDialect implements DbDialect {
}
}
let modifySql = baseSql + modifyArr.join(' ') + ';';
let dropSql = baseSql + ` DROP (${dropArr.join(',')}) ;`;
let modifySql = modifyArr.length > 0 ? baseSql + modifyArr.join(' ') + ';' : '';
let dropSql = dropArr.length > 0 ? baseSql + ` DROP (${dropArr.join(',')}) ;` : '';
let renameSql = renameArr.join('');
let addPkSql = priArr.size > 0 ? `ALTER TABLE ${dbTable} ADD CONSTRAINT "PK_${tableName}" PRIMARY KEY (${Array.from(priArr).join(',')});` : '';
let commentSql = commentArr.join(';');
@@ -473,11 +484,51 @@ class OracleDialect implements DbDialect {
}
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (DbInst.isNumber(columnType)) {
return value;
}
if (value && this.getDataType(columnType) === DataType.DateTime) {
return `to_timestamp('${value}', 'yyyy-mm-dd hh24:mi:ss')`;
}
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
if (duplicateStrategy == DuplicateStrategy.REPLACE) {
// 字段数组生成占位符sql
let phs = [];
let values = [];
let insertFields = [];
for (let i = 0; i < fieldArr.length; i++) {
phs.push(`:${i + 1} ${fieldArr[i]}`);
values.push(`T2.${fieldArr[i]}`);
insertFields.push(`T1.${fieldArr[i]}`);
}
let placeholder = phs.join(',');
let sql = `MERGE INTO ${this.quoteIdentifier(tableName)} T1 USING
(
SELECT ${placeholder} FROM dual
) T2 ON (T1.id = T2.id)
WHEN NOT MATCHED THEN INSERT (${insertFields.join(',')}) VALUES (${values.join(',')})
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
return sql;
} else {
// 字段数组生成占位符sql
let phs = [];
for (let i = 0; i < fieldArr.length; i++) {
phs.push(`:${i + 1} ${fieldArr[i]}`);
}
let ignore = '';
if (duplicateStrategy == DuplicateStrategy.IGNORE) {
ignore = `/*+ IGNORE_ROW_ON_DUPKEY_INDEX(${tableName}(id)) */`;
}
let placeholder = phs.join(',');
return `INSERT ${ignore} INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder});`;
}
}
}

View File

@@ -4,6 +4,7 @@ import {
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
@@ -439,8 +440,26 @@ class PostgresqlDialect implements DbDialect {
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (DbInst.isNumber(columnType)) {
return value;
}
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
// 构建占位符字符串 "($1, $2, $3 ...)"
let placeholder = fieldArr.map((_, i) => `$${i + 1}`).join(',');
let suffix = '';
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
suffix = ' ON CONFLICT DO NOTHING';
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
suffix = ' ON CONFLICT ON CONSTRAINT {your_constraint_name1} DO UPDATE SET ' + fieldArr.map((a) => `${a}=excluded.${a}`).join(',');
}
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}) \n ${suffix};`;
}
}

View File

@@ -3,6 +3,7 @@ import {
DataType,
DbDialect,
DialectInfo,
DuplicateStrategy,
EditorCompletion,
EditorCompletionItem,
IndexDefinition,
@@ -331,8 +332,25 @@ class SqliteDialect implements DbDialect {
}
return DataType.String;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
wrapStrValue(columnType: string, value: string): string {
wrapValue(columnType: string, value: any): any {
if (value == null) {
return 'NULL';
}
if (DbInst.isNumber(columnType)) {
return value;
}
return `'${value}'`;
}
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
let prefix = 'insert into';
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
prefix = 'insert or ignore into';
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
prefix = 'insert or replace into';
}
return `${prefix} ${this.quoteIdentifier(tableName)}(${fieldArr.join(',')}) values (${placeholder});`;
}
}

View File

@@ -20,6 +20,7 @@ type DataSyncTaskForm struct {
TargetTagPath string `binding:"required" json:"targetTagPath"`
TargetTableName string `binding:"required" json:"targetTableName"`
FieldMap string `binding:"required" json:"fieldMap"`
DuplicateStrategy int `json:"duplicateStrategy"`
}
type DataSyncTaskStatusForm struct {

View File

@@ -53,6 +53,7 @@ type dataSyncAppImpl struct {
var (
dateTimeReg = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$`)
whereReg = regexp.MustCompile(`(?i)where`)
)
func (app *dataSyncAppImpl) InjectDbDataSyncTaskRepo(repo repository.DataSyncTask) {
@@ -143,11 +144,15 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
updSql := ""
orderSql := ""
if task.UpdFieldVal != "0" && task.UpdFieldVal != "" && task.UpdField != "" {
srcConn, _ := app.dbApp.GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
srcConn, err := app.dbApp.GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
if err != nil {
logx.Errorf("数据源连接不可用, %s", err.Error())
return
}
task.UpdFieldVal = strings.Trim(task.UpdFieldVal, " ")
// 把UpdFieldVal尝试转为int如果可以转为int则不添加引号否则添加引号
if _, err := strconv.Atoi(task.UpdFieldVal); err != nil {
if _, err = strconv.Atoi(task.UpdFieldVal); err != nil {
updSql = fmt.Sprintf("and %s > '%s'", task.UpdField, task.UpdFieldVal)
} else {
updSql = fmt.Sprintf("and %s > %s", task.UpdField, task.UpdFieldVal)
@@ -162,8 +167,14 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
}
orderSql = "order by " + task.UpdField + " asc "
}
// 正则判断DataSql是否以where .*结尾如果是则不添加where 1 = 1
var where = "where 1=1"
if whereReg.MatchString(task.DataSql) {
where = ""
}
// 组装查询sql
sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s", task.DataSql, updSql, orderSql)
sql := fmt.Sprintf("%s %s %s %s", task.DataSql, where, updSql, orderSql)
log, err := app.doDataSync(sql, task)
if err != nil {
@@ -223,6 +234,12 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
result := make([]map[string]any, 0)
var queryColumns []*dbi.QueryColumn
// 如果有数据库别名则从UpdField中去掉数据库别名, 如a.id => id用于获取字段具体名称
updFieldName := task.UpdField
if task.UpdField != "" && strings.Contains(task.UpdField, ".") {
updFieldName = strings.Split(task.UpdField, ".")[1]
}
err = srcConn.WalkQueryRows(context.Background(), sql, func(row map[string]any, columns []*dbi.QueryColumn) error {
if len(queryColumns) == 0 {
queryColumns = columns
@@ -230,7 +247,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
// 遍历columns 取task.UpdField的字段类型
updFieldType = dbi.DataTypeString
for _, column := range columns {
if strings.EqualFold(strings.ToLower(column.Name), strings.ToLower(task.UpdField)) {
if strings.EqualFold(column.Name, updFieldName) {
updFieldType = srcDialect.GetDataConverter().GetDataType(column.Type)
break
}
@@ -240,7 +257,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
total++
result = append(result, row)
if total%batchSize == 0 {
if err := app.srcData2TargetDb(result, fieldMap, columns, updFieldType, task, srcDialect, targetConn, targetDbTx); err != nil {
if err := app.srcData2TargetDb(result, fieldMap, columns, updFieldType, updFieldName, task, srcDialect, targetConn, targetDbTx); err != nil {
return err
}
@@ -262,15 +279,19 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
// 处理剩余的数据
if len(result) > 0 {
if err := app.srcData2TargetDb(result, fieldMap, queryColumns, updFieldType, task, srcDialect, targetConn, targetDbTx); err != nil {
if err := app.srcData2TargetDb(result, fieldMap, queryColumns, updFieldType, updFieldName, task, srcDialect, targetConn, targetDbTx); err != nil {
targetDbTx.Rollback()
return syncLog, err
}
}
// 如果是mssql暂不手动提交事务否则报错 mssql: The COMMIT TRANSACTION request has no corresponding BEGIN TRANSACTION.
if err := targetDbTx.Commit(); err != nil {
if targetConn.Info.Type != dbi.DbTypeMssql {
return syncLog, errorx.NewBiz("数据同步-目标数据库事务提交失败: %s", err.Error())
}
}
logx.Infof("同步任务:[%s],执行完毕,保存记录成功:[%d]条", task.TaskName, total)
// 保存执行成功日志
@@ -282,7 +303,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
return syncLog, nil
}
func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, columns []*dbi.QueryColumn, updFieldType dbi.DataType, task *entity.DataSyncTask, srcDialect dbi.Dialect, targetDbConn *dbi.DbConn, targetDbTx *sql.Tx) error {
func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, columns []*dbi.QueryColumn, updFieldType dbi.DataType, updFieldName string, task *entity.DataSyncTask, srcDialect dbi.Dialect, targetDbConn *dbi.DbConn, targetDbTx *sql.Tx) error {
// 遍历src字段列表取出字段对应的类型
var srcColumnTypes = make(map[string]string)
@@ -305,9 +326,9 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
data = append(data, rowData)
}
// 解决字段大小写问题
updFieldVal := srcRes[len(srcRes)-1][strings.ToUpper(task.UpdField)]
updFieldVal := srcRes[len(srcRes)-1][strings.ToUpper(updFieldName)]
if updFieldVal == "" || updFieldVal == nil {
updFieldVal = srcRes[len(srcRes)-1][strings.ToLower(task.UpdField)]
updFieldVal = srcRes[len(srcRes)-1][strings.ToLower(updFieldName)]
}
task.UpdFieldVal = srcDialect.GetDataConverter().FormatData(updFieldVal, updFieldType)
@@ -338,7 +359,7 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
}
// 目标数据库执行sql批量插入
_, err := targetDbConn.GetDialect().BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values)
_, err := targetDbConn.GetDialect().BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values, task.DuplicateStrategy)
if err != nil {
return err
}

View File

@@ -61,6 +61,8 @@ func (dbType DbType) RemoveQuote(name string) string {
return removeQuote(name, "`")
case DbTypePostgres, DbTypeGauss, DbTypeKingbaseEs, DbTypeVastbase:
return removeQuote(name, `"`)
case DbTypeMssql:
return strings.Trim(name, "[]")
default:
return removeQuote(name, `"`)
}

View File

@@ -20,6 +20,15 @@ const (
DataTypeDateTime DataType = "datetime"
)
const (
// -1. 无操作
DuplicateStrategyNone = -1
// 1. 忽略
DuplicateStrategyIgnore = 1
// 2. 更新
DuplicateStrategyUpdate = 2
)
// 数据库服务实例信息
type DbServer struct {
Version string `json:"version"` // 版本信息
@@ -111,7 +120,7 @@ type Dialect interface {
GetDbProgram() (DbProgram, error)
// 批量保存数据
BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error)
BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error)
// 获取数据转换器用于解析格式化列数据等
GetDataConverter() DataConverter

View File

@@ -3,7 +3,7 @@ package dbi
import "database/sql"
var (
metas map[DbType]Meta = make(map[DbType]Meta)
metas = make(map[DbType]Meta)
)
// 注册数据库类型与dbmeta

View File

@@ -261,35 +261,114 @@ var (
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
)
func (dd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
func (dd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
// 执行批量insert sql
// insert into "table_name" ("column1", "column2", ...) values (value1, value2, ...)
dbType := dd.dc.Info.Type
// 无需处理重复数据直接执行批量insert
if duplicateStrategy == dbi.DuplicateStrategyNone || duplicateStrategy == 0 {
return dd.batchInsertSimple(dbType, tx, tableName, columns, values)
} else { // 执行MERGE INTO语句
return dd.batchInsertMerge(dbType, tx, tableName, columns, values)
}
}
func (dd *DMDialect) batchInsertSimple(dbType dbi.DbType, tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
// 生成占位符字符串:如:(?,?)
// 重复字符串并用逗号连接
repeated := strings.Repeat("?,", len(columns))
// 去除最后一个逗号,占位符由括号包裹
placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", dd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", dbType.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
effRows := 0
for _, value := range values {
// 达梦数据库只能一条条的执行insert
er, err := dd.dc.TxExec(tx, sqlTemp, value...)
res, err := dd.dc.TxExec(tx, sqlTemp, value...)
if err != nil {
logx.Errorf("执行sql失败%s", err.Error())
return int64(effRows), err
logx.Errorf("执行sql失败%s, sql: [ %s ]", err.Error(), sqlTemp)
}
effRows += int(er)
effRows += int(res)
}
// 执行批量insert sql
return int64(effRows), nil
}
func (dd *DMDialect) batchInsertMerge(dbType dbi.DbType, tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
// 查询主键字段
uniqueCols := make([]string, 0)
caseSqls := make([]string, 0)
tableCols, _ := dd.GetColumns(tableName)
identityCols := make([]string, 0)
for _, col := range tableCols {
if col.IsPrimaryKey {
uniqueCols = append(uniqueCols, col.ColumnName)
caseSqls = append(caseSqls, fmt.Sprintf("( T1.%s = T2.%s )", dbType.QuoteIdentifier(col.ColumnName), dbType.QuoteIdentifier(col.ColumnName)))
}
if col.IsIdentity {
// 自增字段不放入insert内即使是设置了identity_insert on也不起作用
identityCols = append(identityCols, dbType.QuoteIdentifier(col.ColumnName))
}
}
// 查询唯一索引涉及到的字段并组装到match条件内
indexs, _ := dd.GetTableIndex(tableName)
if indexs != nil {
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
tmp := make([]string, 0)
for _, col := range cols {
uniqueCols = append(uniqueCols, col)
tmp = append(tmp, fmt.Sprintf(" T1.%s = T2.%s ", dbType.QuoteIdentifier(col), dbType.QuoteIdentifier(col)))
}
caseSqls = append(caseSqls, fmt.Sprintf("( %s )", strings.Join(tmp, " AND ")))
}
}
}
// 重复数据处理策略
phs := make([]string, 0)
insertVals := make([]string, 0)
upds := make([]string, 0)
insertCols := make([]string, 0)
for _, column := range columns {
phs = append(phs, fmt.Sprintf("? %s", column))
if !collx.ArrayContains(uniqueCols, dbType.RemoveQuote(column)) {
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
}
if !collx.ArrayContains(identityCols, column) {
insertCols = append(insertCols, fmt.Sprintf("%s", column))
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
}
}
t2s := make([]string, 0)
for i := 0; i < len(values); i++ {
t2s = append(t2s, fmt.Sprintf("SELECT %s FROM dual", strings.Join(phs, ",")))
}
t2 := strings.Join(t2s, " UNION ALL ")
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON " + strings.Join(caseSqls, " OR ")
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ")"
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
// 把二维数组转为一维数组
var args []any
for _, v := range values {
args = append(args, v...)
}
return dd.dc.TxExec(tx, sqlTemp, args...)
}
func (dd *DMDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
type DataConverter struct {
@@ -328,6 +407,22 @@ func (dd *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
}
func (dd *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}

View File

@@ -135,13 +135,11 @@ func (md *MssqlDialect) GetPrimaryKey(tablename string) (string, error) {
return columns[0].ColumnName, nil
}
// 获取表索引信息
func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
func (md *MssqlDialect) getTableIndexWithPK(tableName string) ([]dbi.Index, error) {
_, res, err := md.dc.Query(dbi.GetLocalSql(MSSQL_META_FILE, MSSQL_INDEX_INFO_KEY), md.currentSchema(), tableName)
if err != nil {
return nil, err
}
indexs := make([]dbi.Index, 0)
for _, re := range res {
indexs = append(indexs, dbi.Index{
@@ -153,18 +151,14 @@ func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
SeqInIndex: anyx.ConvInt(re["seqInIndex"]),
})
}
// 把查询结果以索引名分组,索引字段以逗号连接
// 把查询结果以索引名分组,多个索引字段以逗号连接
result := make([]dbi.Index, 0)
key := ""
for _, v := range indexs {
// 当前的索引名
in := v.IndexName
// 过滤掉主键索引,主键索引名为PK__开头的
if strings.HasPrefix(in, "PK__") {
continue
}
if key == in {
// 索引字段已根据名称和顺序排序,故取最后一个即可
// 索引字段已根据名称和字段顺序排序,故取最后一个即可
i := len(result) - 1
// 同索引字段以逗号连接
result[i].ColumnName = result[i].ColumnName + "," + v.ColumnName
@@ -173,6 +167,20 @@ func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
result = append(result, v)
}
}
return indexs, nil
}
// 获取表索引信息
func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
indexs, _ := md.getTableIndexWithPK(tableName)
result := make([]dbi.Index, 0)
// 过滤掉主键索引,主键索引名为PK__开头的
for _, v := range indexs {
in := v.IndexName
if strings.HasPrefix(in, "PK__") {
continue
}
}
return result, nil
}
@@ -289,8 +297,39 @@ func (md *MssqlDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", md.dc.Info.Type)
}
func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
if duplicateStrategy == dbi.DuplicateStrategyUpdate {
return md.batchInsertMerge(tx, tableName, columns, values, duplicateStrategy)
}
return md.batchInsertSimple(tx, tableName, columns, values, duplicateStrategy)
}
func (md *MssqlDialect) batchInsertSimple(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
schema := md.currentSchema()
ignoreDupSql := ""
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
// ALTER TABLE dbo.TEST ADD CONSTRAINT uniqueRows UNIQUE (ColA, ColB, ColC, ColD) WITH (IGNORE_DUP_KEY = ON)
indexs, _ := md.getTableIndexWithPK(tableName)
// 收集唯一索引涉及到的字段
uniqueColumns := make([]string, 0)
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
for _, col := range cols {
if !collx.ArrayContains(uniqueColumns, col) {
uniqueColumns = append(uniqueColumns, col)
}
}
}
}
if len(uniqueColumns) > 0 {
// 设置忽略重复键
ignoreDupSql = fmt.Sprintf("ALTER TABLE %s.%s ADD CONSTRAINT uniqueRows UNIQUE (%s) WITH (IGNORE_DUP_KEY = {sign})", schema, tableName, strings.Join(uniqueColumns, ","))
_, _ = md.dc.TxExec(tx, strings.ReplaceAll(ignoreDupSql, "{sign}", "ON"))
}
}
// 生成占位符字符串:如:(?,?)
// 重复字符串并用逗号连接
@@ -312,18 +351,86 @@ func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
for _, v := range values {
args = append(args, v...)
}
// 设置允许填充自增列之后,显示指定列名可以插入自增列
_, _ = md.dc.Exec(fmt.Sprintf("set identity_insert \"%s\" on", tableName))
identityInsertOn := fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, tableName)
exec, err := md.dc.TxExec(tx, sqlStr, args...)
res, err := md.dc.TxExec(tx, fmt.Sprintf("%s %s", identityInsertOn, sqlStr), args...)
_, _ = md.dc.Exec(fmt.Sprintf("set identity_insert \"%s\" off", tableName))
// 执行完之后,设置忽略重复键
if ignoreDupSql != "" {
_, _ = md.dc.TxExec(tx, strings.ReplaceAll(ignoreDupSql, "{sign}", "OFF"))
}
return res, err
}
return exec, err
func (md *MssqlDialect) batchInsertMerge(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
schema := md.currentSchema()
dbType := md.dc.Info.Type
// 收集MERGE 语句的 ON 子句条件
caseSqls := make([]string, 0)
pkCols := make([]string, 0)
// 查询取出自增列字段, merge update不能修改自增列
identityCols := make([]string, 0)
cols, err := md.GetColumns(tableName)
for _, col := range cols {
if col.IsIdentity {
identityCols = append(identityCols, col.ColumnName)
}
if col.IsPrimaryKey {
pkCols = append(pkCols, col.ColumnName)
name := dbType.QuoteIdentifier(col.ColumnName)
caseSqls = append(caseSqls, fmt.Sprintf(" T1.%s = T2.%s ", name, name))
}
}
if len(pkCols) == 0 {
return md.batchInsertSimple(tx, tableName, columns, values, duplicateStrategy)
}
// 重复数据处理策略
insertVals := make([]string, 0)
upds := make([]string, 0)
insertCols := make([]string, 0)
// 源数据占位sql
phs := make([]string, 0)
for _, column := range columns {
if !collx.ArrayContains(identityCols, dbType.RemoveQuote(column)) {
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
}
insertCols = append(insertCols, fmt.Sprintf("%s", column))
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
phs = append(phs, fmt.Sprintf("? %s", column))
}
// 把二维数组转为一维数组
var args []any
tmp := fmt.Sprintf("select %s", strings.Join(phs, ","))
t2s := make([]string, 0)
for _, v := range values {
args = append(args, v...)
t2s = append(t2s, tmp)
}
t2 := strings.Join(t2s, " UNION ALL ")
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(schema) + "." + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON " + strings.Join(caseSqls, " AND ")
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ") "
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
// 设置允许填充自增列之后,显示指定列名可以插入自增列
identityInsertOn := fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, tableName)
// 执行merge sql,必须要以分号结尾
res, err := md.dc.TxExec(tx, fmt.Sprintf("%s %s;", identityInsertOn, sqlTemp), args...)
if err != nil {
logx.Errorf("执行sql失败%s, sql: [ %s ]", err.Error(), sqlTemp)
}
return res, err
}
func (md *MssqlDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
var (
@@ -335,6 +442,8 @@ var (
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
)
type DataConverter struct {
@@ -364,6 +473,20 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
@@ -409,12 +532,10 @@ func (md *MssqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
// 复制数据
// 设置允许填充自增列之后,显示指定列名可以插入自增列
identityInsertOn := ""
identityInsertOff := ""
if hasIdentity {
identityInsertOn = fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, newTableName)
identityInsertOff = fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] OFF", schema, newTableName)
}
_, err = md.dc.Exec(fmt.Sprintf(" %s INSERT INTO [%s].[%s] (%s) SELECT * FROM [%s].[%s] %s", identityInsertOn, schema, newTableName, columnsSql, schema, copy.TableName, identityInsertOff))
_, err = md.dc.Exec(fmt.Sprintf(" %s INSERT INTO [%s].[%s] (%s) SELECT * FROM [%s].[%s]", identityInsertOn, schema, newTableName, columnsSql, schema, copy.TableName))
if err != nil {
logx.Warnf("复制表[%s]数据失败: %s", copy.TableName, err.Error())
}

View File

@@ -10,14 +10,14 @@ import (
)
func init() {
meta := new(Meta)
meta := new(MssqlMeta)
dbi.Register(dbi.DbTypeMssql, meta)
}
type Meta struct {
type MssqlMeta struct {
}
func (md *Meta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
func (md *MssqlMeta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
err := d.IfUseSshTunnelChangeIpPort()
if err != nil {
return nil, err
@@ -52,6 +52,6 @@ func (md *Meta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
return sql.Open(driverName, dsn)
}
func (md *Meta) GetDialect(conn *dbi.DbConn) dbi.Dialect {
func (md *MssqlMeta) GetDialect(conn *dbi.DbConn) dbi.Dialect {
return &MssqlDialect{conn}
}

View File

@@ -45,7 +45,6 @@ func (md *MysqlDialect) GetDbNames() ([]string, error) {
for _, re := range res {
databases = append(databases, anyx.ConvString(re["dbname"]))
}
return databases, nil
}
@@ -173,7 +172,7 @@ func (md *MysqlDialect) GetDbProgram() (dbi.DbProgram, error) {
return NewDbProgramMysql(md.dc), nil
}
func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
// 生成占位符字符串:如:(?,?)
// 重复字符串并用逗号连接
repeated := strings.Repeat("?,", len(columns))
@@ -188,7 +187,14 @@ func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
// 去除最后一个逗号
placeholder = strings.TrimSuffix(repeated, ",")
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
prefix := "insert into"
if duplicateStrategy == 1 {
prefix = "insert ignore into"
} else if duplicateStrategy == 2 {
prefix = "replace into"
}
sqlStr := fmt.Sprintf("%s %s (%s) values %s", prefix, md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
// 执行批量insert sql
// 把二维数组转为一维数组
var args []any
@@ -199,7 +205,7 @@ func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
}
func (md *MysqlDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
var (
@@ -211,6 +217,8 @@ var (
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
)
type DataConverter struct {
@@ -240,6 +248,22 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.DateTime, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
@@ -252,13 +253,7 @@ func (od *OracleDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", od.dc.Info.Type)
}
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
//INSERT ALL
//INTO my_table(field_1,field_2) VALUES (value_1,value_2)
//INTO my_table(field_1,field_2) VALUES (value_3,value_4)
//INTO my_table(field_1,field_2) VALUES (value_5,value_6)
//SELECT 1 FROM DUAL;
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
if len(values) <= 0 {
return 0, nil
}
@@ -269,27 +264,122 @@ func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
args = append(args, v...)
}
// 拼接oracle批量插入语句
sqlArr := make([]string, 0)
sqlArr = append(sqlArr, "INSERT ALL")
if duplicateStrategy == dbi.DuplicateStrategyNone || duplicateStrategy == 0 || duplicateStrategy == dbi.DuplicateStrategyIgnore {
return od.batchInsertSimple(od.dc.Info.Type, tableName, columns, values, duplicateStrategy, tx)
} else {
return od.batchInsertMergeSql(od.dc.Info.Type, tableName, columns, values, args, tx)
}
}
// 简单批量插入sql无需判断键冲突策略
func (od *OracleDialect) batchInsertSimple(dbType dbi.DbType, tableName string, columns []string, values [][]any, duplicateStrategy int, tx *sql.Tx) (int64, error) {
// 忽略键冲突策略
ignore := ""
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
// 查出唯一索引涉及的字段
indexs, _ := od.GetTableIndex(tableName)
if indexs != nil {
arr := make([]string, 0)
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
for _, col := range cols {
if !collx.ArrayContains(arr, col) {
arr = append(arr, col)
}
}
}
}
ignore = fmt.Sprintf("/*+ IGNORE_ROW_ON_DUPKEY_INDEX(%s(%s)) */", tableName, strings.Join(arr, ","))
}
}
effRows := 0
for _, value := range values {
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
var placeholder []string
for i := 0; i < len(value); i++ {
placeholder = append(placeholder, fmt.Sprintf(":%d", i+1))
}
sqlTemp := fmt.Sprintf("INSERT %s INTO %s (%s) VALUES (%s)", ignore, dbType.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ","))
// oracle数据库为了兼容ignore主键冲突只能一条条的执行insert
res, err := od.dc.TxExec(tx, sqlTemp, value...)
if err != nil {
logx.Errorf("执行sql失败%s, sql: [ %s ]", err.Error(), sqlTemp)
}
effRows += int(res)
}
return int64(effRows), nil
}
func (od *OracleDialect) batchInsertMergeSql(dbType dbi.DbType, tableName string, columns []string, values [][]any, args []any, tx *sql.Tx) (int64, error) {
// 查询主键字段
uniqueCols := make([]string, 0)
caseSqls := make([]string, 0)
// 查询唯一索引涉及到的字段并组装到match条件内
indexs, _ := od.GetTableIndex(tableName)
if indexs != nil {
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
tmp := make([]string, 0)
for _, col := range cols {
if !collx.ArrayContains(uniqueCols, col) {
uniqueCols = append(uniqueCols, col)
}
tmp = append(tmp, fmt.Sprintf(" T1.%s = T2.%s ", dbType.QuoteIdentifier(col), dbType.QuoteIdentifier(col)))
}
caseSqls = append(caseSqls, fmt.Sprintf("( %s )", strings.Join(tmp, " AND ")))
}
}
}
// 如果caseSqls为空则说明没有唯一键直接使用简单批量插入
if len(caseSqls) == 0 {
return od.batchInsertSimple(dbType, tableName, columns, values, dbi.DuplicateStrategyNone, tx)
}
// 重复数据处理策略
insertVals := make([]string, 0)
upds := make([]string, 0)
insertCols := make([]string, 0)
for _, column := range columns {
if !collx.ArrayContains(uniqueCols, dbType.RemoveQuote(column)) {
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
}
insertCols = append(insertCols, fmt.Sprintf("T1.%s", column))
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
}
// 生成源数据占位sql
t2s := make([]string, 0)
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
for i := 0; i < len(args); i += len(columns) {
var placeholder []string
for j := 0; j < len(columns); j++ {
placeholder = append(placeholder, fmt.Sprintf(":%d", i+j+1))
col := columns[j]
placeholder = append(placeholder, fmt.Sprintf(":%d %s", i+j+1, col))
}
sqlArr = append(sqlArr, fmt.Sprintf("INTO %s (%s) VALUES (%s)", od.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ",")))
t2s = append(t2s, fmt.Sprintf("SELECT %s FROM dual", strings.Join(placeholder, ", ")))
}
sqlArr = append(sqlArr, "SELECT 1 FROM DUAL")
t2 := strings.Join(t2s, " UNION ALL ")
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON (" + strings.Join(caseSqls, " OR ") + ") "
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ") "
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
// 执行批量insert sql
res, err := od.dc.TxExec(tx, strings.Join(sqlArr, " "), args...)
res, err := od.dc.TxExec(tx, sqlTemp, args...)
if err != nil {
logx.Errorf("执行sql失败%s, sql: [ %s ]", err.Error(), sqlTemp)
}
return res, err
}
func (od *OracleDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
var (
@@ -297,6 +387,8 @@ var (
numberTypeRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeTypeRegexp = regexp.MustCompile(`(?i)date|timestamp`)
converter = new(DataConverter)
)
type DataConverter struct {

View File

@@ -50,7 +50,7 @@ func (md *OraMeta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
}
}
}
urlOptions["TIMEOUT"] = "10"
urlOptions["TIMEOUT"] = "1000"
connStr := go_ora.BuildUrl(d.Host, d.Port, d.Sid, d.Username, d.Password, urlOptions)
conn, err := sql.Open(driverName, connStr)
if err != nil {

View File

@@ -25,8 +25,8 @@ type PgsqlDialect struct {
dc *dbi.DbConn
}
func (md *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
_, res, err := md.dc.Query("SELECT version() as server_version")
func (pd *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
_, res, err := pd.dc.Query("SELECT version() as server_version")
if err != nil {
return nil, err
}
@@ -36,8 +36,8 @@ func (md *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
return ds, nil
}
func (md *PgsqlDialect) GetDbNames() ([]string, error) {
_, res, err := md.dc.Query("SELECT datname AS dbname FROM pg_database WHERE datistemplate = false AND has_database_privilege(datname, 'CONNECT')")
func (pd *PgsqlDialect) GetDbNames() ([]string, error) {
_, res, err := pd.dc.Query("SELECT datname AS dbname FROM pg_database WHERE datistemplate = false AND has_database_privilege(datname, 'CONNECT')")
if err != nil {
return nil, err
}
@@ -51,8 +51,8 @@ func (md *PgsqlDialect) GetDbNames() ([]string, error) {
}
// 获取表基础元信息, 如表名等
func (md *PgsqlDialect) GetTables() ([]dbi.Table, error) {
_, res, err := md.dc.Query(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_INFO_KEY))
func (pd *PgsqlDialect) GetTables() ([]dbi.Table, error) {
_, res, err := pd.dc.Query(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_INFO_KEY))
if err != nil {
return nil, err
}
@@ -72,13 +72,13 @@ func (md *PgsqlDialect) GetTables() ([]dbi.Table, error) {
}
// 获取列元信息, 如列名等
func (md *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
dbType := md.dc.Info.Type
func (pd *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
dbType := pd.dc.Info.Type
tableName := strings.Join(collx.ArrayMap[string, string](tableNames, func(val string) string {
return fmt.Sprintf("'%s'", dbType.RemoveQuote(val))
}), ",")
_, res, err := md.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_COLUMN_MA_KEY), tableName))
_, res, err := pd.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_COLUMN_MA_KEY), tableName))
if err != nil {
return nil, err
}
@@ -100,8 +100,8 @@ func (md *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
return columns, nil
}
func (md *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
columns, err := md.GetColumns(tablename)
func (pd *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
columns, err := pd.GetColumns(tablename)
if err != nil {
return "", err
}
@@ -118,8 +118,8 @@ func (md *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
}
// 获取表索引信息
func (md *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
_, res, err := md.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_INDEX_INFO_KEY), tableName))
func (pd *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
_, res, err := pd.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_INDEX_INFO_KEY), tableName))
if err != nil {
return nil, err
}
@@ -155,17 +155,14 @@ func (md *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
}
// 获取建表ddl
func (md *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
_, err := md.dc.Exec(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_DDL_KEY))
func (pd *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
_, err := pd.dc.Exec(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_DDL_KEY))
if err != nil {
return "", err
}
_, schemaRes, _ := md.dc.Query("select current_schema() as schema")
schemaName := schemaRes[0]["schema"].(string)
ddlSql := fmt.Sprintf("select showcreatetable('%s','%s') as sql", schemaName, tableName)
_, res, err := md.dc.Query(ddlSql)
ddlSql := fmt.Sprintf("select showcreatetable('%s','%s') as sql", pd.currentSchema(), tableName)
_, res, err := pd.dc.Query(ddlSql)
if err != nil {
return "", err
}
@@ -174,9 +171,9 @@ func (md *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
}
// 获取pgsql当前连接的库可访问的schemaNames
func (md *PgsqlDialect) GetSchemas() ([]string, error) {
func (pd *PgsqlDialect) GetSchemas() ([]string, error) {
sql := dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_DB_SCHEMAS)
_, res, err := md.dc.Query(sql)
_, res, err := pd.dc.Query(sql)
if err != nil {
return nil, err
}
@@ -188,11 +185,11 @@ func (md *PgsqlDialect) GetSchemas() ([]string, error) {
}
// GetDbProgram 获取数据库程序模块,用于数据库备份与恢复
func (md *PgsqlDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", md.dc.Info.Type)
func (pd *PgsqlDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", pd.dc.Info.Type)
}
func (md *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
func (pd *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
// 执行批量insert sql跟mysql一样 pg或高斯支持批量insert语法
// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
@@ -212,14 +209,99 @@ func (md *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
placeholders = append(placeholders, "("+strings.Join(placeholder, ", ")+")")
}
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholders, ", "))
// 根据冲突策略生成后缀
suffix := ""
if pd.dc.Info.Type == dbi.DbTypeGauss {
// 高斯db使用ON DUPLICATE KEY UPDATE 语法参考 https://support.huaweicloud.com/distributed-devg-v3-gaussdb/gaussdb-12-0607.html#ZH-CN_TOPIC_0000001633948138
suffix = pd.gaussOnDuplicateStrategySql(duplicateStrategy, tableName, columns)
} else {
// pgsql 默认使用 on conflict 语法参考 http://www.postgres.cn/docs/12/sql-insert.html
// vastbase语法参考 https://docs.vastdata.com.cn/zh/docs/VastbaseE100Ver3.0.0/doc/SQL%E8%AF%AD%E6%B3%95/INSERT.html
// kingbase语法参考 https://help.kingbase.com.cn/v8/development/sql-plsql/sql/SQL_Statements_9.html#insert
suffix = pd.pgsqlOnDuplicateStrategySql(duplicateStrategy, tableName, columns)
}
sqlStr := fmt.Sprintf("insert into %s (%s) values %s %s", pd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholders, ", "), suffix)
// 执行批量insert sql
return md.dc.TxExec(tx, sqlStr, args...)
return pd.dc.TxExec(tx, sqlStr, args...)
}
func (md *PgsqlDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
// pgsql默认唯一键冲突策略
func (pd PgsqlDialect) pgsqlOnDuplicateStrategySql(duplicateStrategy int, tableName string, columns []string) string {
suffix := ""
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
suffix = " \n on conflict do nothing"
} else if duplicateStrategy == dbi.DuplicateStrategyUpdate {
// 生成 on conflict () do update set column1 = excluded.column1, column2 = excluded.column2, ...
var updateColumns []string
for _, col := range columns {
updateColumns = append(updateColumns, fmt.Sprintf("%s = excluded.%s", col, col))
}
// 查询唯一键名,拼接冲突sql
_, keyRes, _ := pd.dc.Query("SELECT constraint_name FROM information_schema.table_constraints WHERE constraint_schema = $1 AND table_name = $2 AND constraint_type in ('PRIMARY KEY', 'UNIQUE') ", pd.currentSchema(), tableName)
if len(keyRes) > 0 {
for _, re := range keyRes {
key := anyx.ToString(re["constraint_name"])
if key != "" {
suffix += fmt.Sprintf(" \n on conflict on constraint %s do update set %s \n", key, strings.Join(updateColumns, ", "))
}
}
}
}
return suffix
}
// 高斯db唯一键冲突策略,使用ON DUPLICATE KEY UPDATE 参考https://support.huaweicloud.com/distributed-devg-v3-gaussdb/gaussdb-12-0607.html#ZH-CN_TOPIC_0000001633948138
func (pd PgsqlDialect) gaussOnDuplicateStrategySql(duplicateStrategy int, tableName string, columns []string) string {
suffix := ""
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
suffix = " \n ON DUPLICATE KEY UPDATE NOTHING"
} else if duplicateStrategy == dbi.DuplicateStrategyUpdate {
// 查出表里的唯一键涉及的字段
var uniqueColumns []string
indexs, err := pd.GetTableIndex(tableName)
if err == nil {
for _, index := range indexs {
if index.IsUnique {
cols := strings.Split(index.ColumnName, ",")
for _, col := range cols {
if !collx.ArrayContains(uniqueColumns, strings.ToLower(col)) {
uniqueColumns = append(uniqueColumns, strings.ToLower(col))
}
}
}
}
}
suffix = " \n ON DUPLICATE KEY UPDATE "
for i, col := range columns {
// ON DUPLICATE KEY UPDATE语句不支持更新唯一键字段所以得去掉
if !collx.ArrayContains(uniqueColumns, pd.dc.Info.Type.RemoveQuote(strings.ToLower(col))) {
suffix += fmt.Sprintf("%s = excluded.%s", col, col)
if i < len(columns)-1 {
suffix += ", "
}
}
}
}
return suffix
}
// 从连接信息中获取数据库和schema信息
func (pd *PgsqlDialect) currentSchema() string {
dbName := pd.dc.Info.Database
schema := ""
arr := strings.Split(dbName, "/")
if len(arr) == 2 {
schema = arr[1]
}
return schema
}
func (pd *PgsqlDialect) GetDataConverter() dbi.DataConverter {
return converter
}
var (
@@ -231,6 +313,8 @@ var (
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
)
type DataConverter struct {
@@ -272,19 +356,33 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (md *PgsqlDialect) IsGauss() bool {
return strings.Contains(md.dc.Info.Params, "gauss")
func (pd *PgsqlDialect) IsGauss() bool {
return strings.Contains(pd.dc.Info.Params, "gauss")
}
func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
func (pd *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
tableName := copy.TableName
// 生成新表名,为老表明+_copy_时间戳
newTableName := tableName + "_copy_" + time.Now().Format("20060102150405")
// 执行根据旧表创建新表
_, err := md.dc.Exec(fmt.Sprintf("create table %s (like %s)", newTableName, tableName))
_, err := pd.dc.Exec(fmt.Sprintf("create table %s (like %s)", newTableName, tableName))
if err != nil {
return err
}
@@ -292,12 +390,12 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
// 复制数据
if copy.CopyData {
go func() {
_, _ = md.dc.Exec(fmt.Sprintf("insert into %s select * from %s", newTableName, tableName))
_, _ = pd.dc.Exec(fmt.Sprintf("insert into %s select * from %s", newTableName, tableName))
}()
}
// 查询旧表的自增字段名 重新设置新表的序列序列器
_, res, err := md.dc.Query(fmt.Sprintf("select column_name from information_schema.columns where table_name = '%s' and column_default like 'nextval%%'", tableName))
_, res, err := pd.dc.Query(fmt.Sprintf("select column_name from information_schema.columns where table_name = '%s' and column_default like 'nextval%%'", tableName))
if err != nil {
return err
}
@@ -307,7 +405,7 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
if colName != "" {
// 查询自增列当前最大值
_, maxRes, err := md.dc.Query(fmt.Sprintf("select max(%s) max_val from %s", colName, tableName))
_, maxRes, err := pd.dc.Query(fmt.Sprintf("select max(%s) max_val from %s", colName, tableName))
if err != nil {
return err
}
@@ -323,12 +421,12 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
newSeqName := fmt.Sprintf("%s_%s_copy_seq", newTableName, colName)
// 创建自增序列,当前最大值为旧表最大值
_, err = md.dc.Exec(fmt.Sprintf("CREATE SEQUENCE %s START %d INCREMENT 1", newSeqName, maxVal))
_, err = pd.dc.Exec(fmt.Sprintf("CREATE SEQUENCE %s START %d INCREMENT 1", newSeqName, maxVal))
if err != nil {
return err
}
// 将新表的自增主键序列与主键列相关联
_, err = md.dc.Exec(fmt.Sprintf("alter table %s alter column %s set default nextval('%s')", newTableName, colName, newSeqName))
_, err = pd.dc.Exec(fmt.Sprintf("alter table %s alter column %s set default nextval('%s')", newTableName, colName, newSeqName))
if err != nil {
return err
}

View File

@@ -20,8 +20,9 @@ func init() {
dbi.Register(dbi.DbTypeKingbaseEs, meta)
dbi.Register(dbi.DbTypeVastbase, meta)
gauss := new(PostgresMeta)
gauss.Param = "dbtype=gauss"
gauss := &PostgresMeta{
Param: "dbtype=gauss",
}
dbi.Register(dbi.DbTypeGauss, gauss)
}

View File

@@ -184,7 +184,7 @@ func (sd *SqliteDialect) GetDbProgram() (dbi.DbProgram, error) {
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", sd.dc.Info.Type)
}
func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
// 执行批量insert sql跟mysql一样 支持批量insert语法
// 生成占位符字符串:如:(?,?)
// 重复字符串并用逗号连接
@@ -197,7 +197,14 @@ func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
// 去除最后一个逗号
placeholder = strings.TrimSuffix(repeated, ",")
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", sd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
prefix := "insert into"
if duplicateStrategy == 1 {
prefix = "insert or ignore into"
} else if duplicateStrategy == 2 {
prefix = "insert or replace into"
}
sqlStr := fmt.Sprintf("%s %s (%s) values %s", prefix, sd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
// 把二维数组转为一维数组
var args []any
@@ -210,7 +217,7 @@ func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
}
func (sd *SqliteDialect) GetDataConverter() dbi.DataConverter {
return new(DataConverter)
return converter
}
var (
@@ -218,6 +225,8 @@ var (
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit|real`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime`)
converter = new(DataConverter)
)
type DataConverter struct {

View File

@@ -31,6 +31,7 @@ type DataSyncTask struct {
TargetTagPath string `orm:"column(target_tag_path)" json:"targetTagPath"`
TargetTableName string `orm:"column(target_table_name)" json:"targetTableName"`
FieldMap string `orm:"column(field_map)" json:"fieldMap"` // 字段映射json
DuplicateStrategy int `orm:"column(duplicate_strategy)" json:"duplicateStrategy"` // 冲突策略 -11忽略2覆盖
}
func (d *DataSyncTask) TableName() string {

View File

@@ -7,7 +7,6 @@ import (
"mayfly-go/pkg/scheduler"
"mayfly-go/pkg/utils/netx"
"net"
"os"
"sync"
"golang.org/x/crypto/ssh"
@@ -86,14 +85,12 @@ func (stm *SshTunnelMachine) OpenSshTunnel(id string, ip string, port int) (expo
return "", 0, err
}
hostname, err := os.Hostname()
if err != nil {
return "", 0, err
}
// debug
//hostname = "0.0.0.0"
localAddr := fmt.Sprintf("%s:%d", hostname, localPort)
localHost := "127.0.0.1"
localAddr := fmt.Sprintf("%s:%d", localHost, localPort)
listener, err := net.Listen("tcp", localAddr)
if err != nil {
return "", 0, err
@@ -102,7 +99,7 @@ func (stm *SshTunnelMachine) OpenSshTunnel(id string, ip string, port int) (expo
tunnel = &Tunnel{
id: id,
machineId: stm.machineId,
localHost: hostname,
localHost: localHost,
localPort: localPort,
remoteHost: ip,
remotePort: port,

View File

@@ -39,6 +39,7 @@ CREATE TABLE IF NOT EXISTS "t_db" (
"modifier" text(32),
"is_deleted" integer(8) NOT NULL,
"delete_time" datetime,
"flow_procdef_key" text(64),
PRIMARY KEY ("id")
);
@@ -164,6 +165,7 @@ CREATE TABLE IF NOT EXISTS "t_db_data_sync_task" (
"recent_state" integer(1) NOT NULL,
"task_key" text(100),
"running_state" integer(1),
"duplicate_strategy" integer(1),
PRIMARY KEY ("id")
);
@@ -267,6 +269,9 @@ CREATE TABLE IF NOT EXISTS "t_db_sql_exec" (
"modifier_id" integer(20) NOT NULL,
"is_deleted" integer(8) NOT NULL,
"delete_time" datetime,
"status" integer(8),
"flow_biz_key" text(128),
"res" text(1000),
PRIMARY KEY ("id")
);
@@ -842,6 +847,12 @@ INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight,
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709045735, 1708910975, '6egfEVYr/3r3hHEub/', 1, 1, '我的任务', 'procinst-tasks', 1708911263, '{"component":"flow/ProcinstTaskList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstTaskList"}', 1, 'admin', 1, 'admin', '2024-02-27 22:55:35', '2024-02-27 22:56:35', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708911264, 1708910975, '6egfEVYr/fw0Hhvye/', 1, 1, '流程定义', 'procdefs', 1708911264, '{"component":"flow/ProcdefList","icon":"List","isKeepAlive":true,"routeName":"ProcdefList"}', 1, 'admin', 1, 'admin', '2024-02-26 09:34:24', '2024-02-27 22:54:32', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708910975, 0, '6egfEVYr/', 1, 1, '工单流程', '/flow', 60000000, '{"icon":"List","isKeepAlive":true,"routeName":"flow"}', 1, 'admin', 1, 'admin', '2024-02-26 09:29:36', '2024-02-26 15:37:52', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709208354, 1708911264, '6egfEVYr/fw0Hhvye/b4cNf3iq/', 2, 1, '删除流程', 'flow:procdef:del', 1709208354, 'null', 1, 'admin', 1, 'admin', '2024-02-29 20:05:54', '2024-02-29 20:05:54', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709208339, 1708911264, '6egfEVYr/fw0Hhvye/r9ZMTHqC/', 2, 1, '保存流程', 'flow:procdef:save', 1709208339, 'null', 1, 'admin', 1, 'admin', '2024-02-29 20:05:40', '2024-02-29 20:05:40', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709103180, 1708910975, '6egfEVYr/oNCIbynR/', 1, 1, '我的流程', 'procinsts', 1708911263, '{"component":"flow/ProcinstList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstList"}', 1, 'admin', 1, 'admin', '2024-02-28 14:53:00', '2024-02-29 20:36:07', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709045735, 1708910975, '6egfEVYr/3r3hHEub/', 1, 1, '我的任务', 'procinst-tasks', 1708911263, '{"component":"flow/ProcinstTaskList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstTaskList"}', 1, 'admin', 1, 'admin', '2024-02-27 22:55:35', '2024-02-27 22:56:35', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708911264, 1708910975, '6egfEVYr/fw0Hhvye/', 1, 1, '流程定义', 'procdefs', 1708911264, '{"component":"flow/ProcdefList","icon":"List","isKeepAlive":true,"routeName":"ProcdefList"}', 1, 'admin', 1, 'admin', '2024-02-26 09:34:24', '2024-02-27 22:54:32', 0, NULL);
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708910975, 0, '6egfEVYr/', 1, 1, '工单流程', '/flow', 60000000, '{"icon":"List","isKeepAlive":true,"routeName":"flow"}', 1, 'admin', 1, 'admin', '2024-02-26 09:29:36', '2024-02-26 15:37:52', 0, NULL);
-- Table: t_sys_role
@@ -988,6 +999,66 @@ CREATE TABLE IF NOT EXISTS "t_team_member" (
);
INSERT INTO t_team_member (id, team_id, account_id, username, create_time, creator_id, creator, update_time, modifier_id, modifier, is_deleted, delete_time) VALUES (7, 3, 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', 0, NULL);
CREATE TABLE "t_flow_procdef" (
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
"def_key" text(100) NOT NULL ,
"name" text(200) DEFAULT NULL ,
"status" tinyint DEFAULT NULL ,
"tasks" text(3000) NOT NULL ,
"remark" text(191) DEFAULT NULL,
"create_time" datetime NOT NULL,
"creator" text(191) DEFAULT NULL,
"creator_id" integer(20) NOT NULL,
"update_time" datetime NOT NULL,
"modifier" text(191) DEFAULT NULL,
"modifier_id" integer(20) NOT NULL,
"is_deleted" tinyint DEFAULT '0',
"delete_time" datetime DEFAULT NULL
) ;
CREATE TABLE "t_flow_procinst" (
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
"procdef_id" integer(20) NOT NULL ,
"procdef_name" text(64) NOT NULL ,
"task_key" text(100) DEFAULT NULL ,
"status" tinyint DEFAULT NULL ,
"biz_type" text(64) NOT NULL ,
"biz_key" text(64) NOT NULL ,
"biz_status" tinyint DEFAULT NULL ,
"biz_handle_res" text(100) DEFAULT NULL ,
"remark" text(191) DEFAULT NULL,
"end_time" datetime DEFAULT NULL ,
"duration" integer(20) DEFAULT NULL ,
"create_time" datetime NOT NULL ,
"creator" text(191) DEFAULT NULL ,
"creator_id" integer(20) NOT NULL,
"update_time" datetime NOT NULL,
"modifier" text(191) DEFAULT NULL,
"modifier_id" integer(20) NOT NULL,
"is_deleted" tinyint DEFAULT '0',
"delete_time" datetime DEFAULT NULL
) ;
CREATE TABLE "t_flow_procinst_task" (
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
"procinst_id" integer(20) NOT NULL ,
"task_key" text(64) NOT NULL ,
"task_name" text(64) DEFAULT NULL ,
"assignee" text(64) DEFAULT NULL ,
"status" tinyint DEFAULT NULL ,
"remark" text(191) DEFAULT NULL,
"end_time" datetime DEFAULT NULL ,
"duration" integer(20) DEFAULT NULL ,
"create_time" datetime NOT NULL ,
"creator" text(191) DEFAULT NULL,
"creator_id" integer(20) NOT NULL,
"update_time" datetime NOT NULL,
"modifier" text(191) DEFAULT NULL,
"modifier_id" integer(20) NOT NULL,
"is_deleted" tinyint DEFAULT '0',
"delete_time" datetime DEFAULT NULL
)
-- Index: idx_db_backup_id
CREATE INDEX IF NOT EXISTS "idx_db_backup_id"
ON "t_db_backup_history" (
@@ -1006,5 +1077,17 @@ ON "t_db_backup" (
"db_name" ASC
);
-- Index: idx_procdef_id
CREATE INDEX IF NOT EXISTS "idx_procdef_id"
ON "t_flow_procinst" (
"procdef_id" ASC
);
-- Index: idx_procinst_id
CREATE INDEX IF NOT EXISTS "idx_procinst_id"
ON "t_flow_procinst_task" (
"procinst_id" ASC
);
COMMIT TRANSACTION;
PRAGMA foreign_keys = on;

View File

@@ -282,6 +282,7 @@ CREATE TABLE `t_db_data_sync_task`
`recent_state` tinyint(1) NOT NULL DEFAULT '0' COMMENT '最近一次状态 0未执行 1成功 2失败',
`task_key` varchar(100) DEFAULT NULL COMMENT '定时任务唯一uuid key',
`running_state` tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止',
`duplicate_strategy`tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -11忽略2覆盖',
PRIMARY KEY (`id`)
) COMMENT='数据同步';

View File

@@ -11,68 +11,68 @@ INSERT INTO t_sys_resource (id, pid, ui_path, `type`, status, name, code, weight
-- 工单流程相关表
CREATE TABLE `t_flow_procdef` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`def_key` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '流程定义key',
`name` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '流程名称',
`def_key` varchar(100) NOT NULL COMMENT '流程定义key',
`name` varchar(191) DEFAULT NULL COMMENT '流程名称',
`status` tinyint DEFAULT NULL COMMENT '状态',
`tasks` varchar(3000) COLLATE utf8mb4_bin NOT NULL COMMENT '审批节点任务信息',
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`remark` varchar(191) DEFAULT NULL,
`create_time` datetime NOT NULL,
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`creator` varchar(191) DEFAULT NULL,
`creator_id` bigint NOT NULL,
`update_time` datetime NOT NULL,
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`modifier` varchar(191) DEFAULT NULL,
`modifier_id` bigint NOT NULL,
`is_deleted` tinyint DEFAULT '0',
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程定义';
) COMMENT='流程-流程定义';
CREATE TABLE `t_flow_procinst` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`procdef_id` bigint NOT NULL COMMENT '流程定义id',
`procdef_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '流程定义名称',
`task_key` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '当前任务key',
`procdef_name` varchar(64) NOT NULL COMMENT '流程定义名称',
`task_key` varchar(100) DEFAULT NULL COMMENT '当前任务key',
`status` tinyint DEFAULT NULL COMMENT '状态',
`biz_type` varchar(64) COLLATE utf8mb4_bin NOT NULL COMMENT '关联业务类型',
`biz_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '关联业务key',
`biz_key` varchar(64) NOT NULL COMMENT '关联业务key',
`biz_status` tinyint DEFAULT NULL COMMENT '业务状态',
`biz_handle_res` varchar(100) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '关联的业务处理结果',
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`remark` varchar(191) DEFAULT NULL,
`end_time` datetime DEFAULT NULL COMMENT '结束时间',
`duration` bigint DEFAULT NULL COMMENT '流程持续时间(开始到结束)',
`create_time` datetime NOT NULL COMMENT '流程发起时间',
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '流程发起人',
`creator` varchar(191) DEFAULT NULL COMMENT '流程发起人',
`creator_id` bigint NOT NULL,
`update_time` datetime NOT NULL,
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`modifier` varchar(191) DEFAULT NULL,
`modifier_id` bigint NOT NULL,
`is_deleted` tinyint DEFAULT '0',
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_procdef_id` (`procdef_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程实例(根据流程定义开启一个流程)';
) COMMENT='流程-流程实例(根据流程定义开启一个流程)';
CREATE TABLE `t_flow_procinst_task` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`procinst_id` bigint NOT NULL COMMENT '流程实例id',
`task_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '任务key',
`task_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '任务名称',
`assignee` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '分配到该任务的用户',
`task_key` varchar(64) NOT NULL COMMENT '任务key',
`task_name` varchar(64) DEFAULT NULL COMMENT '任务名称',
`assignee` varchar(64) DEFAULT NULL COMMENT '分配到该任务的用户',
`status` tinyint DEFAULT NULL COMMENT '状态',
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`remark` varchar(191) DEFAULT NULL,
`end_time` datetime DEFAULT NULL COMMENT '结束时间',
`duration` bigint DEFAULT NULL COMMENT '任务持续时间(开始到结束)',
`create_time` datetime NOT NULL COMMENT '任务开始时间',
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`creator` varchar(191) DEFAULT NULL,
`creator_id` bigint NOT NULL,
`update_time` datetime NOT NULL,
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`modifier` varchar(191) DEFAULT NULL,
`modifier_id` bigint NOT NULL,
`is_deleted` tinyint DEFAULT '0',
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_procinst_id` (`procinst_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程实例任务';
) COMMENT='流程-流程实例任务';
-- 新增工单流程相关字段
ALTER TABLE t_db_sql_exec ADD status tinyint NULL COMMENT '执行状态';
@@ -82,6 +82,9 @@ ALTER TABLE t_db_sql_exec ADD res varchar(1000) NULL COMMENT '执行结果';
ALTER TABLE t_db ADD flow_procdef_key varchar(64) NULL COMMENT '审批流-流程定义key有值则说明关键操作需要进行审批执行';
-- 历史执行记录调整为成功状态
UPDATE t_db_sql_exec SET status = 2
UPDATE t_db_sql_exec SET status = 2;
ALTER TABLE `t_db_data_sync_task`
ADD COLUMN `duplicate_strategy` tinyint(1) NULL DEFAULT -1 comment '冲突策略 -11忽略2覆盖';
commit;