mirror of
https://gitee.com/dromara/mayfly-go
synced 2025-11-02 23:40:24 +08:00
!106 feat:数据同步支持唯一键冲突策略
* refactor:sql同步 * fix: 表格右键导出菜单换行符修复 * feat:数据同步支持唯一键冲突策略
This commit is contained in:
@@ -7,24 +7,22 @@ export function exportCsv(filename: string, columns: string[], datas: []) {
|
||||
for (let column of columns) {
|
||||
let val: any = data[column];
|
||||
if (val == null || val == undefined) {
|
||||
dataValueArr.push('');
|
||||
continue;
|
||||
}
|
||||
val = '';
|
||||
} else if (val && typeof val == 'string') {
|
||||
// 替换换行符
|
||||
val = val.replace(/[\r\n]/g, '\\n');
|
||||
|
||||
if (typeof val == 'string' && val) {
|
||||
// csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
if (val.indexOf(',') != -1) {
|
||||
// 如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if (val.indexOf('"') != -1) {
|
||||
val = val.replace(/\"/g, '""');
|
||||
val = val.replace(/"/g, '""');
|
||||
}
|
||||
// 再将逗号转义
|
||||
val = `"${val}"`;
|
||||
}
|
||||
dataValueArr.push(val + '\t');
|
||||
} else {
|
||||
dataValueArr.push(val + '\t');
|
||||
}
|
||||
dataValueArr.push(String(val));
|
||||
}
|
||||
cvsData.push(dataValueArr);
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ import { ElMessage } from 'element-plus';
|
||||
import TagTreeSelect from '../component/TagTreeSelect.vue';
|
||||
import type { CheckboxValueType } from 'element-plus';
|
||||
import ProcdefSelectFormItem from '@/views/flow/components/ProcdefSelectFormItem.vue';
|
||||
import { DbType } from '@/views/ops/db/dialect';
|
||||
|
||||
const props = defineProps({
|
||||
visible: {
|
||||
@@ -196,7 +197,14 @@ const changeInstance = () => {
|
||||
|
||||
const getAllDatabase = async () => {
|
||||
if (state.form.instanceId > 0) {
|
||||
state.allDatabases = await dbApi.getAllDatabase.request({ instanceId: state.form.instanceId });
|
||||
let dbs = await dbApi.getAllDatabase.request({ instanceId: state.form.instanceId });
|
||||
state.allDatabases = dbs;
|
||||
|
||||
// 如果是oracle,且没查出数据库列表,则取实例sid
|
||||
let instance = state.instances.find((item: any) => item.id === state.form.instanceId);
|
||||
if (instance && instance.type === DbType.oracle && dbs.length === 0) {
|
||||
state.allDatabases = [instance.sid];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@
|
||||
</template>
|
||||
|
||||
<script lang="ts" setup>
|
||||
import { ref, toRefs, reactive, onMounted, defineAsyncComponent, Ref } from 'vue';
|
||||
import { defineAsyncComponent, onMounted, reactive, ref, Ref, toRefs } from 'vue';
|
||||
import { ElMessage, ElMessageBox } from 'element-plus';
|
||||
import { dbApi } from './api';
|
||||
import { dateFormat } from '@/common/utils/date';
|
||||
|
||||
@@ -685,8 +685,9 @@ const onDeleteTable = async (data: any) => {
|
||||
cancelButtonText: '取消',
|
||||
type: 'warning',
|
||||
});
|
||||
|
||||
// 执行sql
|
||||
dbApi.sqlExec.request({ id, db, sql: `drop table ${tableName}` }).then(() => {
|
||||
dbApi.sqlExec.request({ id, db, sql: `drop table ${getDbDialect(state.nowDbInst.type).quoteIdentifier(tableName)}` }).then(() => {
|
||||
if (flowProcdefKey) {
|
||||
ElMessage.success('工单提交成功');
|
||||
return;
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<el-row>
|
||||
<el-col :span="11">
|
||||
<el-form-item prop="taskName" label="任务名" required>
|
||||
<el-input v-model.trim="form.taskName" placeholder="请输入数据库别名" auto-complete="off" />
|
||||
<el-input v-model.trim="form.taskName" placeholder="请输入同步任务名" auto-complete="off" />
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
|
||||
@@ -89,9 +89,11 @@
|
||||
</el-col>
|
||||
|
||||
<el-col :span="8">
|
||||
<el-tooltip content="查询数据源的时候会带上这个字段当前最大值,支持带别名,如:t.create_time" placement="top">
|
||||
<el-form-item prop="updField" label="更新字段" required>
|
||||
<el-input v-model.trim="form.updField" placeholder="查询数据源的时候会带上这个字段当前最大值" auto-complete="off" />
|
||||
</el-form-item>
|
||||
</el-tooltip>
|
||||
</el-col>
|
||||
|
||||
<el-col :span="8">
|
||||
@@ -125,10 +127,17 @@
|
||||
|
||||
<el-tab-pane label="sql预览" :name="sqlPreviewTab" :disabled="!baseFieldCompleted">
|
||||
<el-form-item prop="fieldMap" label="查询sql">
|
||||
<el-input type="textarea" v-model="state.previewDataSql" readonly :input-style="{ height: '190px' }" />
|
||||
<el-input type="textarea" v-model="state.previewDataSql" readonly :input-style="{ height: '170px' }" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="fieldMap" label="插入sql">
|
||||
<el-input type="textarea" v-model="state.previewInsertSql" readonly :input-style="{ height: '190px' }" />
|
||||
<el-input type="textarea" v-model="state.previewInsertSql" readonly :input-style="{ height: '170px' }" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="isReplace" v-if="compatibleDuplicateStrategy(form.targetDbType!)" label="键冲突策略">
|
||||
<el-select v-model="form.duplicateStrategy" @change="handleDuplicateStrategy" style="width: 100px">
|
||||
<el-option label="无" :value="DuplicateStrategy.NONE" />
|
||||
<el-option label="忽略" :value="DuplicateStrategy.IGNORE" />
|
||||
<el-option label="替换" :value="DuplicateStrategy.REPLACE" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</el-tab-pane>
|
||||
</el-tabs>
|
||||
@@ -185,7 +194,7 @@ import { ElMessage } from 'element-plus';
|
||||
import DbSelectTree from '@/views/ops/db/component/DbSelectTree.vue';
|
||||
import MonacoEditor from '@/components/monaco/MonacoEditor.vue';
|
||||
import { DbInst, registerDbCompletionItemProvider } from '@/views/ops/db/db';
|
||||
import { DbType, getDbDialect } from '@/views/ops/db/dialect';
|
||||
import { compatibleDuplicateStrategy, DbType, DuplicateStrategy, getDbDialect } from '@/views/ops/db/dialect';
|
||||
import CrontabInput from '@/components/crontab/CrontabInput.vue';
|
||||
|
||||
const props = defineProps({
|
||||
@@ -246,6 +255,7 @@ type FormData = {
|
||||
updFieldVal?: string;
|
||||
fieldMap?: { src: string; target: string }[];
|
||||
status?: 1 | 2;
|
||||
duplicateStrategy?: -1 | 1 | 2;
|
||||
};
|
||||
|
||||
const basicFormData = {
|
||||
@@ -257,6 +267,7 @@ const basicFormData = {
|
||||
updFieldVal: '0',
|
||||
fieldMap: [{ src: 'a', target: 'b' }],
|
||||
status: 1,
|
||||
duplicateStrategy: -1,
|
||||
} as FormData;
|
||||
|
||||
const state = reactive({
|
||||
@@ -271,6 +282,7 @@ const state = reactive({
|
||||
previewRes: {} as any,
|
||||
previewDataSql: '',
|
||||
previewInsertSql: '',
|
||||
previewFieldArr: [] as string[],
|
||||
});
|
||||
|
||||
const { tabActiveName, form, submitForm } = toRefs(state);
|
||||
@@ -295,6 +307,9 @@ watch(dialogVisible, async (newValue: boolean) => {
|
||||
|
||||
let data = await dbApi.getDatasyncTask.request({ taskId: propsData?.id });
|
||||
state.form = data;
|
||||
if (!state.form.duplicateStrategy) {
|
||||
state.form.duplicateStrategy = -1;
|
||||
}
|
||||
try {
|
||||
state.form.fieldMap = JSON.parse(data.fieldMap);
|
||||
} catch (e) {
|
||||
@@ -343,13 +358,12 @@ watch(tabActiveName, async (newValue: string) => {
|
||||
await handleGetTargetFields();
|
||||
break;
|
||||
case sqlPreviewTab:
|
||||
let srcDbDialect = getDbDialect(state.srcDbInst.type);
|
||||
let targetDbDialect = getDbDialect(state.targetDbInst.type);
|
||||
let updField = state.form.updField!;
|
||||
|
||||
let updField = srcDbDialect.quoteIdentifier(state.form.updField!);
|
||||
state.previewDataSql = `SELECT * FROM (\n ${state.form.dataSql?.trim() || '请输入数据sql'} \n ) t \n where ${updField} > '${
|
||||
state.form.updFieldVal || ''
|
||||
}'`;
|
||||
// 判断sql是否以where .*结尾
|
||||
let hasCondition = /where/i.test(state.form.dataSql!);
|
||||
state.previewDataSql = `${state.form.dataSql?.trim() || '请输入数据sql'} \n ${hasCondition ? 'and' : 'where'} ${updField} > '${state.form.updFieldVal || ''}'`;
|
||||
|
||||
// 检查字段映射中是否存在重复的目标字段
|
||||
let fields = new Set();
|
||||
@@ -365,17 +379,19 @@ watch(tabActiveName, async (newValue: string) => {
|
||||
}
|
||||
|
||||
let fieldArr = state.form.fieldMap?.map((a: any) => targetDbDialect.quoteIdentifier(a.target)) || [];
|
||||
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
|
||||
|
||||
state.previewInsertSql = ` insert into ${targetDbDialect.quoteIdentifier(state.form.targetTableName!)}(${fieldArr.join(
|
||||
','
|
||||
)}) values (${placeholder});`;
|
||||
state.previewFieldArr = fieldArr;
|
||||
refreshPreviewInsertSql();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
const refreshPreviewInsertSql = () => {
|
||||
let targetDbDialect = getDbDialect(state.targetDbInst.type);
|
||||
state.previewInsertSql = targetDbDialect.getBatchInsertPreviewSql(state.form.targetTableName!, state.previewFieldArr, state.form.duplicateStrategy!);
|
||||
};
|
||||
|
||||
const onSelectSrcDb = async (params: any) => {
|
||||
// 初始化数据源
|
||||
params.databases = params.dbs; // 数据源里需要这个值
|
||||
@@ -422,16 +438,24 @@ const handleGetSrcFields = async () => {
|
||||
}
|
||||
|
||||
// 执行sql
|
||||
let sql: string;
|
||||
|
||||
if (state.form.srcDbType === DbType.mssql) {
|
||||
// mssql的分页语法不一样
|
||||
let top1 = `select top 1`;
|
||||
sql = `${top1} * from (${state.form.dataSql}) a`;
|
||||
} else if (state.form.srcDbType === DbType.oracle) {
|
||||
// oracle的分页关键字不一样
|
||||
let limit = ' limit 1';
|
||||
if (state.form.srcDbType === DbType.oracle) {
|
||||
limit = ' where rownum <= 1';
|
||||
let hasCondition = /where/i.test(state.form.dataSql!);
|
||||
sql = `${state.form.dataSql} ${hasCondition ? 'and' : 'where'} rownum <= 1`;
|
||||
} else {
|
||||
sql = `${state.form.dataSql} limit 1`;
|
||||
}
|
||||
|
||||
const res = await dbApi.sqlExec.request({
|
||||
id: state.form.srcDbId,
|
||||
db: state.form.srcDbName,
|
||||
sql: `select * from (${state.form.dataSql}) t ${limit}`,
|
||||
sql,
|
||||
});
|
||||
|
||||
if (!res.columns) {
|
||||
@@ -505,6 +529,11 @@ const btnOk = async () => {
|
||||
const cancel = () => {
|
||||
dialogVisible.value = false;
|
||||
emit('cancel');
|
||||
state.form = basicFormData;
|
||||
};
|
||||
|
||||
const handleDuplicateStrategy = () => {
|
||||
refreshPreviewInsertSql();
|
||||
};
|
||||
</script>
|
||||
<style lang="scss">
|
||||
|
||||
@@ -742,6 +742,7 @@ const onExitEditMode = (rowData: any, column: any, rowIndex = 0) => {
|
||||
};
|
||||
|
||||
const submitUpdateFields = async () => {
|
||||
debugger;
|
||||
const dbInst = getNowDbInst();
|
||||
if (cellUpdateMap.size == 0) {
|
||||
return;
|
||||
|
||||
@@ -37,6 +37,7 @@ import { ref, watch, onMounted } from 'vue';
|
||||
import ColumnFormItem from './ColumnFormItem.vue';
|
||||
import { DbInst } from '../../db';
|
||||
import { ElMessage } from 'element-plus';
|
||||
import { getDbDialect } from '@/views/ops/db/dialect';
|
||||
|
||||
export interface ColumnFormItemProps {
|
||||
dbInst: DbInst;
|
||||
|
||||
@@ -524,7 +524,7 @@ const onConfirmCondition = () => {
|
||||
}
|
||||
const row = conditionDialog.columnRow as any;
|
||||
condition += `${row.columnName} ${conditionDialog.condition} `;
|
||||
state.condition = condition + DbInst.wrapColumnValue(row.columnType, conditionDialog.value);
|
||||
state.condition = condition + state.dbDialect.wrapValue(row.columnType, conditionDialog.value!);
|
||||
onCancelCondition();
|
||||
condInputRef.value.focus();
|
||||
};
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
|
||||
<el-select v-else-if="item.prop === 'type'" filterable size="small" v-model="scope.row.type">
|
||||
<el-option
|
||||
v-for="pgsqlType in state.columnTypeList"
|
||||
v-for="pgsqlType in getDbDialect(dbType).getInfo().columnTypes"
|
||||
:key="pgsqlType.dataType"
|
||||
:value="pgsqlType.udtName"
|
||||
:label="pgsqlType.dataType"
|
||||
@@ -172,7 +172,6 @@ const state = reactive({
|
||||
dialogVisible: false,
|
||||
btnloading: false,
|
||||
activeName: '1',
|
||||
columnTypeList: dbDialect.getInfo().columnTypes,
|
||||
tableData: {
|
||||
fields: {
|
||||
colNames: [
|
||||
@@ -347,22 +346,25 @@ const submit = async () => {
|
||||
* @param nowArr 修改后的对象数组
|
||||
* @param key 标志对象唯一属性
|
||||
*/
|
||||
const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { del: any[]; add: any[]; upd: any[] } => {
|
||||
const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { del: any[]; add: any[]; upd: any[]; changed: boolean } => {
|
||||
let data = {
|
||||
del: [] as object[], // 删除的数据
|
||||
add: [] as object[], // 新增的数据
|
||||
upd: [] as object[], // 修改的数据
|
||||
changed: false,
|
||||
};
|
||||
|
||||
// 旧数据为空
|
||||
if (oldArr && Array.isArray(oldArr) && oldArr.length === 0 && nowArr && Array.isArray(nowArr) && nowArr.length > 0) {
|
||||
data.add = nowArr;
|
||||
data.changed = true;
|
||||
return data;
|
||||
}
|
||||
|
||||
// 新数据为空
|
||||
if (nowArr && Array.isArray(nowArr) && nowArr.length === 0 && oldArr && Array.isArray(oldArr) && oldArr.length > 0) {
|
||||
data.del = oldArr;
|
||||
data.changed = true;
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -378,6 +380,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
|
||||
oldName && (newMap[oldName] = a);
|
||||
if (!oldMap.hasOwnProperty(k) && (!oldName || (oldName && !oldMap.hasOwnProperty(oldName)))) {
|
||||
// 新增
|
||||
data.changed = true;
|
||||
data.add.push(a);
|
||||
}
|
||||
});
|
||||
@@ -387,6 +390,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
|
||||
let newData = newMap[k];
|
||||
if (!newData) {
|
||||
// 删除
|
||||
data.changed = true;
|
||||
data.del.push(a);
|
||||
} else {
|
||||
// 判断每个字段是否相等,否则为修改
|
||||
@@ -394,6 +398,7 @@ const filterChangedData = (oldArr: object[], nowArr: object[], key: string): { d
|
||||
let oldV = a[f];
|
||||
let newV = newData[f];
|
||||
if (oldV?.toString() !== newV?.toString()) {
|
||||
data.changed = true;
|
||||
data.upd.push(newData);
|
||||
break;
|
||||
}
|
||||
@@ -416,13 +421,17 @@ const genSql = () => {
|
||||
} else {
|
||||
// 修改列
|
||||
let changeColData = filterChangedData(state.tableData.fields.oldFields, state.tableData.fields.res, 'name');
|
||||
let colSql = dbDialect.getModifyColumnSql(data, data.tableName, changeColData);
|
||||
let colSql = changeColData.changed ? dbDialect.getModifyColumnSql(data, data.tableName, changeColData) : '';
|
||||
// 修改索引
|
||||
let changeIdxData = filterChangedData(state.tableData.indexs.oldIndexs, state.tableData.indexs.res, 'indexName');
|
||||
let idxSql = dbDialect.getModifyIndexSql(data, data.tableName, changeIdxData);
|
||||
let idxSql = changeColData.changed ? dbDialect.getModifyIndexSql(data, data.tableName, changeIdxData) : '';
|
||||
// 修改表名
|
||||
|
||||
return colSql + ';' + idxSql;
|
||||
let sqlArr = [];
|
||||
colSql && sqlArr.push(colSql);
|
||||
idxSql && sqlArr.push(idxSql);
|
||||
|
||||
return sqlArr.join(';');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -432,7 +441,9 @@ const reset = () => {
|
||||
state.tableData.tableName = '';
|
||||
state.tableData.tableComment = '';
|
||||
state.tableData.fields.res = [];
|
||||
state.tableData.fields.oldFields = [];
|
||||
state.tableData.indexs.res = [];
|
||||
state.tableData.indexs.oldIndexs = [];
|
||||
};
|
||||
|
||||
const indexChanges = (row: any) => {
|
||||
|
||||
@@ -166,7 +166,7 @@ export class DbInst {
|
||||
const db = this.getDb(dbName);
|
||||
// 优先从 table map中获取
|
||||
let columns = db.getColumns(table);
|
||||
if (columns) {
|
||||
if (columns && columns.length > 0) {
|
||||
return columns;
|
||||
}
|
||||
console.log(`load columns -> dbName: ${dbName}, table: ${table}`);
|
||||
@@ -258,7 +258,7 @@ export class DbInst {
|
||||
|
||||
// 获取指定表的默认查询sql
|
||||
getDefaultSelectSql(db: string, table: string, condition: string, orderBy: string, pageNum: number, limit: number = DbInst.DefaultLimit) {
|
||||
return getDbDialect(this.type).getDefaultSelectSql(db, table, condition, orderBy, pageNum, limit);
|
||||
return this.getDialect().getDefaultSelectSql(db, table, condition, orderBy, pageNum, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -266,12 +266,22 @@ export class DbInst {
|
||||
* @param dbName 数据库名
|
||||
* @param table 表名
|
||||
* @param datas 要生成的数据
|
||||
* @param dbDialect db方言
|
||||
* @param skipNull 是否跳过空字段
|
||||
*/
|
||||
async genInsertSql(dbName: string, table: string, datas: any[], skipNull = false) {
|
||||
if (!datas) {
|
||||
return '';
|
||||
}
|
||||
let schema = '';
|
||||
let arr = dbName.split('/');
|
||||
if (arr.length == 1) {
|
||||
schema = this.wrapName(dbName) + '.';
|
||||
} else if (arr.length == 2) {
|
||||
schema = this.wrapName(arr[1]) + '.';
|
||||
}
|
||||
|
||||
let dbDialect = this.getDialect();
|
||||
const columns = await this.loadColumns(dbName, table);
|
||||
const sqls = [];
|
||||
for (let data of datas) {
|
||||
@@ -283,9 +293,9 @@ export class DbInst {
|
||||
continue;
|
||||
}
|
||||
colNames.push(this.wrapName(colName));
|
||||
values.push(DbInst.wrapValueByType(data[colName]));
|
||||
values.push(dbDialect.wrapValue(column.dataType, data[colName]));
|
||||
}
|
||||
sqls.push(`INSERT INTO ${this.wrapName(table)} (${colNames.join(', ')}) VALUES(${values.join(', ')})`);
|
||||
sqls.push(`INSERT INTO ${schema}${this.wrapName(table)} (${colNames.join(', ')}) VALUES(${values.join(', ')})`);
|
||||
}
|
||||
return sqls.join(';\n') + ';';
|
||||
}
|
||||
@@ -315,22 +325,23 @@ export class DbInst {
|
||||
const v = columnValue[k];
|
||||
// 更新字段列信息
|
||||
const updateColumn = await this.loadTableColumn(dbName, table, k);
|
||||
sql += ` ${this.wrapName(k)} = ${DbInst.wrapColumnValue(updateColumn.columnType, v, dialect)},`;
|
||||
sql += ` ${this.wrapName(k)} = ${dialect.wrapValue(updateColumn.columnType, v)},`;
|
||||
}
|
||||
|
||||
sql = sql.substring(0, sql.length - 1);
|
||||
return (sql += ` WHERE ${this.wrapName(primaryKeyName)} = ${DbInst.wrapColumnValue(primaryKeyType, primaryKeyValue)} ;`);
|
||||
|
||||
return sql + ` WHERE ${this.wrapName(primaryKeyName)} = ${this.getDialect().wrapValue(primaryKeyType, primaryKeyValue)} ;`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成根据主键删除的sql语句
|
||||
* @param db 数据库名
|
||||
* @param table 表名
|
||||
* @param datas 要删除的记录
|
||||
*/
|
||||
async genDeleteByPrimaryKeysSql(db: string, table: string, datas: any[]) {
|
||||
const primaryKey = await this.loadTableColumn(db, table);
|
||||
const primaryKeyColumnName = primaryKey.columnName;
|
||||
const ids = datas.map((d: any) => `${DbInst.wrapColumnValue(primaryKey.columnType, d[primaryKeyColumnName])}`).join(',');
|
||||
const ids = datas.map((d: any) => `${this.getDialect().wrapValue(primaryKey.columnType, d[primaryKeyColumnName])}`).join(',');
|
||||
return `DELETE FROM ${this.wrapName(table)} WHERE ${this.wrapName(primaryKeyColumnName)} IN (${ids})`;
|
||||
}
|
||||
|
||||
@@ -351,9 +362,8 @@ export class DbInst {
|
||||
|
||||
/**
|
||||
* 包裹数据库表名、字段名等,避免使用关键字为字段名或表名时报错
|
||||
* @param table
|
||||
* @param condition
|
||||
* @returns
|
||||
* @param name 表名、字段名、schema名
|
||||
* @returns 包裹后的字符串
|
||||
*/
|
||||
wrapName = (name: string) => {
|
||||
return this.getDialect().quoteIdentifier(name);
|
||||
@@ -410,34 +420,6 @@ export class DbInst {
|
||||
dbInstCache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据返回值包装值,若值为字符串类型则添加''
|
||||
* @param val 值
|
||||
* @returns 包装后的值
|
||||
*/
|
||||
static wrapValueByType = (val: any) => {
|
||||
if (val == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (typeof val == 'number') {
|
||||
return val;
|
||||
}
|
||||
return `'${val}'`;
|
||||
};
|
||||
|
||||
/**
|
||||
* 根据字段类型包装字段值,如为字符串等则添加‘’,数字类型则直接返回即可
|
||||
*/
|
||||
static wrapColumnValue(columnType: string, value: any, dbDialect?: DbDialect) {
|
||||
if (this.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
if (!dbDialect) {
|
||||
return `'${value}'`;
|
||||
}
|
||||
return dbDialect.wrapStrValue(columnType, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断字段类型是否为数字类型
|
||||
* @param columnType 字段类型
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
@@ -682,8 +683,54 @@ class DMDialect implements DbDialect {
|
||||
return DataType.String;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (DbInst.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
// 替换
|
||||
// MERGE INTO t_person T1
|
||||
// USING (
|
||||
// <foreach collection="list" item="item" index="index" separator="UNION ALL">
|
||||
// SELECT
|
||||
// #{item.id} id,
|
||||
// #{item.mc} mc,
|
||||
// #{item.sex} sex,
|
||||
// #{item.age} age
|
||||
// FROM dual
|
||||
// </foreach>
|
||||
// ) T2 ON (T1.id = T2.id )
|
||||
// WHEN NOT MATCHED THEN INSERT(id, mc, sex,
|
||||
// age) VALUES
|
||||
// (T2.id, T2.mc, T2.sex, T2.age)
|
||||
// WHEN MATCHED THEN UPDATE
|
||||
// SET T1.mc = T2.mc,T1.sex = T2.sex,T1.age = T2.age
|
||||
|
||||
if (duplicateStrategy == DuplicateStrategy.REPLACE || duplicateStrategy == DuplicateStrategy.IGNORE) {
|
||||
// 字段数组生成占位符sql
|
||||
let phs = [];
|
||||
let values = [];
|
||||
for (let i = 0; i < fieldArr.length; i++) {
|
||||
phs.push(`? ${fieldArr[i]}`);
|
||||
values.push(`T2.${fieldArr[i]}`);
|
||||
}
|
||||
let placeholder = phs.join(',');
|
||||
let sql = `MERGE INTO ${tableName} T1 USING
|
||||
(
|
||||
SELECT ${placeholder} FROM dual
|
||||
) T2 ON (T1.id = T2.id)
|
||||
WHEN NOT MATCHED THEN INSERT(${fieldArr.join(',')}) VALUES (${values.join(',')})
|
||||
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
|
||||
return sql;
|
||||
} else {
|
||||
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
|
||||
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}), (${placeholder});`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { PostgresqlDialect } from '@/views/ops/db/dialect/postgres_dialect';
|
||||
import { DialectInfo } from '@/views/ops/db/dialect/index';
|
||||
import { DialectInfo, DuplicateStrategy } from '@/views/ops/db/dialect/index';
|
||||
|
||||
let gsDialectInfo: DialectInfo;
|
||||
export class GaussDialect extends PostgresqlDialect {
|
||||
@@ -14,4 +14,17 @@ export class GaussDialect extends PostgresqlDialect {
|
||||
gsDialectInfo.name = 'GaussDB';
|
||||
return gsDialectInfo;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
// 构建占位符字符串 "($1, $2, $3 ...)"
|
||||
let placeholder = fieldArr.map((_, i) => `$${i + 1}`).join(',');
|
||||
let suffix = '';
|
||||
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
|
||||
suffix = '\nON DUPLICATE KEY UPDATE NOTHING';
|
||||
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
|
||||
suffix = '\n-- 执行前会删除唯一键涉及到的字段 \nON DUPLICATE KEY UPDATE ' + fieldArr.map((a) => `${a}=excluded.${a}`).join(',');
|
||||
}
|
||||
|
||||
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}) ${suffix};`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +146,25 @@ export const compatibleMysql = (dbType: string): boolean => {
|
||||
}
|
||||
};
|
||||
|
||||
// 哪些数据库支持键冲突策略
|
||||
export const compatibleDuplicateStrategy = (dbType: string): boolean => {
|
||||
switch (dbType) {
|
||||
case DbType.mysql:
|
||||
case DbType.mariadb:
|
||||
case DbType.postgresql:
|
||||
case DbType.gauss:
|
||||
case DbType.kingbaseEs:
|
||||
case DbType.vastbase:
|
||||
case DbType.dm:
|
||||
case DbType.oracle:
|
||||
case DbType.sqlite:
|
||||
case DbType.mssql:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export interface DbDialect {
|
||||
/**
|
||||
* 获取一些数据库默认信息
|
||||
@@ -206,8 +225,22 @@ export interface DbDialect {
|
||||
/** 通过数据库字段类型,返回基本数据类型 */
|
||||
getDataType(columnType: string): DataType;
|
||||
|
||||
/** 包装字符串数据, 如:oracle需要把date类型改为 to_date(str, 'yyyy-mm-dd hh24:mi:ss') */
|
||||
wrapStrValue(columnType: string, value: string): string;
|
||||
/** 包装字符串数据, 如:oracle需要把date类型改为 to_date(str, 'yyyy-mm-dd hh24:mi:ss') mssql需要把中文字符串数据包装为 N'中文字符串' */
|
||||
wrapValue(columnType: string, value: any): any;
|
||||
|
||||
/**
|
||||
* 生成插入数据预览sql
|
||||
* @param tableName 表名
|
||||
* @param columns 列名
|
||||
* @param duplicateStrategy 重复策略
|
||||
*/
|
||||
getBatchInsertPreviewSql(tableName: string, columns: string[], duplicateStrategy: DuplicateStrategy): string;
|
||||
}
|
||||
|
||||
export enum DuplicateStrategy {
|
||||
NONE = -1, // 无
|
||||
IGNORE = 1, // 忽略
|
||||
REPLACE = 2, // 覆盖
|
||||
}
|
||||
|
||||
let mysqlDialect = new MysqlDialect();
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import { DbInst } from '../db';
|
||||
import { commonCustomKeywords, DataType, DbDialect, DialectInfo, EditorCompletion, EditorCompletionItem, IndexDefinition, RowDefinition } from './index';
|
||||
import {
|
||||
commonCustomKeywords,
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
RowDefinition,
|
||||
} from './index';
|
||||
import { language as sqlLanguage } from 'monaco-editor/esm/vs/basic-languages/sql/sql.js';
|
||||
|
||||
export { MSSQL_TYPE_LIST, MssqlDialect };
|
||||
@@ -134,7 +144,7 @@ class MssqlDialect implements DbDialect {
|
||||
{ name: 'creator_id', type: 'bigint', length: '20', numScale: '', value: '', notNull: true, pri: false, auto_increment: false, remark: '创建人id' },
|
||||
{
|
||||
name: 'creator',
|
||||
type: 'varchar',
|
||||
type: 'nvarchar',
|
||||
length: '100',
|
||||
numScale: '',
|
||||
value: '',
|
||||
@@ -157,7 +167,7 @@ class MssqlDialect implements DbDialect {
|
||||
{ name: 'updator_id', type: 'bigint', length: '20', numScale: '', value: '', notNull: true, pri: false, auto_increment: false, remark: '修改人id' },
|
||||
{
|
||||
name: 'updator',
|
||||
type: 'varchar',
|
||||
type: 'nvarchar',
|
||||
length: '100',
|
||||
numScale: '',
|
||||
value: '',
|
||||
@@ -263,7 +273,10 @@ class MssqlDialect implements DbDialect {
|
||||
}
|
||||
});
|
||||
|
||||
return sql.join(';') + ';' + indexComment.join(';');
|
||||
let arr = [];
|
||||
sql.length > 0 && arr.push(sql.join(';'));
|
||||
indexComment.length > 0 && arr.push(indexComment.join(';'));
|
||||
return arr.join(';');
|
||||
}
|
||||
|
||||
getModifyColumnSql(tableData: any, tableName: string, changeData: { del: RowDefinition[]; add: RowDefinition[]; upd: RowDefinition[] }): string {
|
||||
@@ -326,14 +339,15 @@ ELSE
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
delSql +
|
||||
(addArr.length > 0 ? addArr.join(';') + ';' : '') +
|
||||
(renameArr.length > 0 ? renameArr.join(';') + ';' : '') +
|
||||
(updArr.length > 0 ? updArr.join(';') + ';' : '') +
|
||||
(changeCommentArr.length > 0 ? changeCommentArr.join(';') + ';' : '') +
|
||||
(addCommentArr.length > 0 ? addCommentArr.join(';') + ';' : '')
|
||||
);
|
||||
let arr = [];
|
||||
delSql && arr.push(delSql);
|
||||
addArr.length > 0 && arr.push(addArr.join(';'));
|
||||
renameArr.length > 0 && arr.push(renameArr.join(';'));
|
||||
updArr.length > 0 && arr.push(updArr.join(';'));
|
||||
changeCommentArr.length > 0 && arr.push(changeCommentArr.join(';'));
|
||||
addCommentArr.length > 0 && arr.push(addCommentArr.join(';'));
|
||||
|
||||
return arr.join(';');
|
||||
}
|
||||
|
||||
getModifyIndexSql(tableData: any, tableName: string, changeData: { del: any[]; add: any[]; upd: any[] }): string {
|
||||
@@ -377,7 +391,12 @@ ELSE
|
||||
let dropSql = dropArr.join(';');
|
||||
let addSql = addArr.join(';');
|
||||
let commentSql = commentArr.join(';');
|
||||
return dropSql + ';' + addSql + ';' + commentSql + ';';
|
||||
|
||||
let arr = [];
|
||||
dropSql && arr.push(dropSql);
|
||||
addSql && arr.push(addSql);
|
||||
commentSql && arr.push(commentSql);
|
||||
return arr.join(';');
|
||||
}
|
||||
|
||||
getDataType(columnType: string): DataType {
|
||||
@@ -398,8 +417,46 @@ ELSE
|
||||
}
|
||||
return DataType.String;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (this.getDataType(columnType) == DataType.Number) {
|
||||
return value;
|
||||
}
|
||||
if (this.getDataType(columnType) == DataType.String) {
|
||||
return `N'${value}'`;
|
||||
}
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
|
||||
let baseSql = `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder});`;
|
||||
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
|
||||
let on = `ALTER TABLE ${tableName} ADD CONSTRAINT uniqueRows UNIQUE (id) WITH (IGNORE_DUP_KEY = ON);`;
|
||||
return on + '\n' + baseSql;
|
||||
}
|
||||
|
||||
if (duplicateStrategy === DuplicateStrategy.REPLACE) {
|
||||
// 字段数组生成占位符sql
|
||||
let phs = [];
|
||||
let values = [];
|
||||
for (let i = 0; i < fieldArr.length; i++) {
|
||||
phs.push(`? ${fieldArr[i]}`);
|
||||
values.push(`T2.${fieldArr[i]}`);
|
||||
}
|
||||
let placeholder = phs.join(',');
|
||||
let sql = `MERGE INTO ${tableName} T1 USING
|
||||
(
|
||||
SELECT ${placeholder}
|
||||
) T2 ON (T1.id = T2.id)
|
||||
WHEN NOT MATCHED THEN INSERT(${fieldArr.join(',')}) VALUES (${values.join(',')})
|
||||
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
|
||||
return sql;
|
||||
}
|
||||
|
||||
return baseSql;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import { DbInst } from '../db';
|
||||
import { commonCustomKeywords, DataType, DbDialect, DialectInfo, EditorCompletion, EditorCompletionItem, IndexDefinition, RowDefinition } from './index';
|
||||
import {
|
||||
commonCustomKeywords,
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
RowDefinition,
|
||||
} from './index';
|
||||
import { language as mysqlLanguage } from 'monaco-editor/esm/vs/basic-languages/mysql/mysql.js';
|
||||
|
||||
export { MYSQL_TYPE_LIST, MysqlDialect };
|
||||
@@ -227,7 +237,6 @@ class MysqlDialect implements DbDialect {
|
||||
}
|
||||
|
||||
getModifyColumnSql(tableData: any, tableName: string, changeData: { del: RowDefinition[]; add: RowDefinition[]; upd: RowDefinition[] }): string {
|
||||
let sql = `ALTER TABLE ${this.quoteIdentifier(tableData.db)}.${this.quoteIdentifier(tableName)}`;
|
||||
let arr = [] as string[];
|
||||
if (changeData.del.length > 0) {
|
||||
changeData.del.forEach((a) => {
|
||||
@@ -250,9 +259,14 @@ class MysqlDialect implements DbDialect {
|
||||
});
|
||||
}
|
||||
|
||||
if (arr.length > 0) {
|
||||
let sql = `ALTER TABLE ${this.quoteIdentifier(tableData.db)}.${this.quoteIdentifier(tableName)}`;
|
||||
return sql + arr.join(',') + ';';
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
getModifyIndexSql(tableData: any, tableName: string, changeData: { del: any[]; add: any[]; upd: any[] }): string {
|
||||
// 搜集修改和删除的索引,添加到drop index xx
|
||||
// 收集新增和修改的索引,添加到ADD xx
|
||||
@@ -327,8 +341,27 @@ class MysqlDialect implements DbDialect {
|
||||
}
|
||||
return DataType.String;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (DbInst.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
// 转义所有的换行符
|
||||
value = value.replace(/[\r\n]/g, '\\n');
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: number): string {
|
||||
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
|
||||
let prefix = 'insert into';
|
||||
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
|
||||
prefix = 'insert ignore into';
|
||||
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
|
||||
prefix = 'replace into';
|
||||
}
|
||||
return `${prefix} ${this.quoteIdentifier(tableName)}(${fieldArr.join(',')}) values (${placeholder});`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
@@ -84,7 +85,7 @@ const replaceFunctions: EditorCompletionItem[] = [
|
||||
{ label: 'CURRENT_TIMESTAMP', insertText: 'TIMESTAMP', description: '获取当前时间' },
|
||||
// 转换函数
|
||||
{ label: 'TO_CHAR', insertText: 'TO_CHAR(d|n[,fmt])', description: '把日期和数字转换为制定格式的字符串' },
|
||||
{ label: 'TO_DATE', insertText: 'TO_DATE(X,[,fmt])', description: '把一个字符串以fmt格式转换成一个日期类型' },
|
||||
{ label: 'TO_DATE', insertText: `TO_DATE(X, 'yyyy-MM-dd HH24:mi:ss')`, description: '把一个字符串以fmt格式转换成一个日期类型' },
|
||||
{ label: 'TO_NUMBER', insertText: 'TO_NUMBER(X,[,fmt])', description: '把一个字符串以fmt格式转换为一个数字' },
|
||||
{ label: 'TO_TIMESTAMP', insertText: 'TO_TIMESTAMP(X,[,fmt])', description: '把一个字符串以fmt格式转换为日期类型' },
|
||||
// 其他
|
||||
@@ -310,6 +311,7 @@ class OracleDialect implements DbDialect {
|
||||
let createSql = '';
|
||||
let tableCommentSql = '';
|
||||
let columCommentSql = '';
|
||||
let pris = [] as string[];
|
||||
|
||||
// 创建表结构
|
||||
let fields: string[] = [];
|
||||
@@ -319,9 +321,18 @@ class OracleDialect implements DbDialect {
|
||||
if (item.remark) {
|
||||
columCommentSql += ` COMMENT ON COLUMN ${dbTable}.${this.quoteIdentifier(item.name)} is '${item.remark}'; `;
|
||||
}
|
||||
// 主键
|
||||
if (item.pri) {
|
||||
pris.push(this.quoteIdentifier(item.name));
|
||||
}
|
||||
});
|
||||
// 主键语句
|
||||
let prisql = '';
|
||||
if (pris.length > 0) {
|
||||
prisql = ` CONSTRAINT "PK_${data.tableName}" PRIMARY KEY (${pris.join(',')});`;
|
||||
}
|
||||
// 建表
|
||||
createSql = `CREATE TABLE ${dbTable} ( ${fields.join(',')} );`;
|
||||
createSql = `CREATE TABLE ${dbTable} ( ${fields.join(',')} ) ${prisql ? ',' + prisql : ''};`;
|
||||
// 表注释
|
||||
if (data.tableComment) {
|
||||
tableCommentSql = ` COMMENT ON TABLE ${dbTable} is '${data.tableComment}'; `;
|
||||
@@ -376,7 +387,7 @@ class OracleDialect implements DbDialect {
|
||||
}
|
||||
modifyArr.push(` MODIFY (${this.genColumnBasicSql(a, false)})`);
|
||||
if (a.pri) {
|
||||
priArr.add(`${this.quoteIdentifier(a.name)}"`);
|
||||
priArr.add(`${this.quoteIdentifier(a.name)}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -412,8 +423,8 @@ class OracleDialect implements DbDialect {
|
||||
}
|
||||
}
|
||||
|
||||
let modifySql = baseSql + modifyArr.join(' ') + ';';
|
||||
let dropSql = baseSql + ` DROP (${dropArr.join(',')}) ;`;
|
||||
let modifySql = modifyArr.length > 0 ? baseSql + modifyArr.join(' ') + ';' : '';
|
||||
let dropSql = dropArr.length > 0 ? baseSql + ` DROP (${dropArr.join(',')}) ;` : '';
|
||||
let renameSql = renameArr.join('');
|
||||
let addPkSql = priArr.size > 0 ? `ALTER TABLE ${dbTable} ADD CONSTRAINT "PK_${tableName}" PRIMARY KEY (${Array.from(priArr).join(',')});` : '';
|
||||
let commentSql = commentArr.join(';');
|
||||
@@ -473,11 +484,51 @@ class OracleDialect implements DbDialect {
|
||||
}
|
||||
return DataType.String;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (DbInst.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
if (value && this.getDataType(columnType) === DataType.DateTime) {
|
||||
return `to_timestamp('${value}', 'yyyy-mm-dd hh24:mi:ss')`;
|
||||
}
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
if (duplicateStrategy == DuplicateStrategy.REPLACE) {
|
||||
// 字段数组生成占位符sql
|
||||
let phs = [];
|
||||
let values = [];
|
||||
let insertFields = [];
|
||||
for (let i = 0; i < fieldArr.length; i++) {
|
||||
phs.push(`:${i + 1} ${fieldArr[i]}`);
|
||||
values.push(`T2.${fieldArr[i]}`);
|
||||
insertFields.push(`T1.${fieldArr[i]}`);
|
||||
}
|
||||
let placeholder = phs.join(',');
|
||||
let sql = `MERGE INTO ${this.quoteIdentifier(tableName)} T1 USING
|
||||
(
|
||||
SELECT ${placeholder} FROM dual
|
||||
) T2 ON (T1.id = T2.id)
|
||||
WHEN NOT MATCHED THEN INSERT (${insertFields.join(',')}) VALUES (${values.join(',')})
|
||||
WHEN MATCHED THEN UPDATE SET ${fieldArr.map((a) => `T1.${a} = T2.${a}`).join(',')}`;
|
||||
return sql;
|
||||
} else {
|
||||
// 字段数组生成占位符sql
|
||||
let phs = [];
|
||||
for (let i = 0; i < fieldArr.length; i++) {
|
||||
phs.push(`:${i + 1} ${fieldArr[i]}`);
|
||||
}
|
||||
let ignore = '';
|
||||
if (duplicateStrategy == DuplicateStrategy.IGNORE) {
|
||||
ignore = `/*+ IGNORE_ROW_ON_DUPKEY_INDEX(${tableName}(id)) */`;
|
||||
}
|
||||
let placeholder = phs.join(',');
|
||||
return `INSERT ${ignore} INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder});`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
@@ -439,8 +440,26 @@ class PostgresqlDialect implements DbDialect {
|
||||
return DataType.String;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (DbInst.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
// 构建占位符字符串 "($1, $2, $3 ...)"
|
||||
let placeholder = fieldArr.map((_, i) => `$${i + 1}`).join(',');
|
||||
let suffix = '';
|
||||
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
|
||||
suffix = ' ON CONFLICT DO NOTHING';
|
||||
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
|
||||
suffix = ' ON CONFLICT ON CONSTRAINT {your_constraint_name1} DO UPDATE SET ' + fieldArr.map((a) => `${a}=excluded.${a}`).join(',');
|
||||
}
|
||||
|
||||
return `INSERT INTO ${tableName} (${fieldArr.join(',')}) VALUES (${placeholder}) \n ${suffix};`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
DataType,
|
||||
DbDialect,
|
||||
DialectInfo,
|
||||
DuplicateStrategy,
|
||||
EditorCompletion,
|
||||
EditorCompletionItem,
|
||||
IndexDefinition,
|
||||
@@ -331,8 +332,25 @@ class SqliteDialect implements DbDialect {
|
||||
}
|
||||
return DataType.String;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars,no-unused-vars
|
||||
wrapStrValue(columnType: string, value: string): string {
|
||||
|
||||
wrapValue(columnType: string, value: any): any {
|
||||
if (value == null) {
|
||||
return 'NULL';
|
||||
}
|
||||
if (DbInst.isNumber(columnType)) {
|
||||
return value;
|
||||
}
|
||||
return `'${value}'`;
|
||||
}
|
||||
|
||||
getBatchInsertPreviewSql(tableName: string, fieldArr: string[], duplicateStrategy: DuplicateStrategy): string {
|
||||
let placeholder = '?'.repeat(fieldArr.length).split('').join(',');
|
||||
let prefix = 'insert into';
|
||||
if (duplicateStrategy === DuplicateStrategy.IGNORE) {
|
||||
prefix = 'insert or ignore into';
|
||||
} else if (duplicateStrategy === DuplicateStrategy.REPLACE) {
|
||||
prefix = 'insert or replace into';
|
||||
}
|
||||
return `${prefix} ${this.quoteIdentifier(tableName)}(${fieldArr.join(',')}) values (${placeholder});`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ type DataSyncTaskForm struct {
|
||||
TargetTagPath string `binding:"required" json:"targetTagPath"`
|
||||
TargetTableName string `binding:"required" json:"targetTableName"`
|
||||
FieldMap string `binding:"required" json:"fieldMap"`
|
||||
DuplicateStrategy int `json:"duplicateStrategy"`
|
||||
}
|
||||
|
||||
type DataSyncTaskStatusForm struct {
|
||||
|
||||
@@ -53,6 +53,7 @@ type dataSyncAppImpl struct {
|
||||
|
||||
var (
|
||||
dateTimeReg = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$`)
|
||||
whereReg = regexp.MustCompile(`(?i)where`)
|
||||
)
|
||||
|
||||
func (app *dataSyncAppImpl) InjectDbDataSyncTaskRepo(repo repository.DataSyncTask) {
|
||||
@@ -143,11 +144,15 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
|
||||
updSql := ""
|
||||
orderSql := ""
|
||||
if task.UpdFieldVal != "0" && task.UpdFieldVal != "" && task.UpdField != "" {
|
||||
srcConn, _ := app.dbApp.GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
|
||||
srcConn, err := app.dbApp.GetDbConn(uint64(task.SrcDbId), task.SrcDbName)
|
||||
if err != nil {
|
||||
logx.Errorf("数据源连接不可用, %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
task.UpdFieldVal = strings.Trim(task.UpdFieldVal, " ")
|
||||
// 把UpdFieldVal尝试转为int,如果可以转为int,则不添加引号,否则添加引号
|
||||
if _, err := strconv.Atoi(task.UpdFieldVal); err != nil {
|
||||
if _, err = strconv.Atoi(task.UpdFieldVal); err != nil {
|
||||
updSql = fmt.Sprintf("and %s > '%s'", task.UpdField, task.UpdFieldVal)
|
||||
} else {
|
||||
updSql = fmt.Sprintf("and %s > %s", task.UpdField, task.UpdFieldVal)
|
||||
@@ -162,8 +167,14 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
|
||||
}
|
||||
orderSql = "order by " + task.UpdField + " asc "
|
||||
}
|
||||
// 正则判断DataSql是否以where .*结尾,如果是则不添加where 1 = 1
|
||||
var where = "where 1=1"
|
||||
if whereReg.MatchString(task.DataSql) {
|
||||
where = ""
|
||||
}
|
||||
|
||||
// 组装查询sql
|
||||
sql := fmt.Sprintf("select * from (%s) t where 1 = 1 %s %s", task.DataSql, updSql, orderSql)
|
||||
sql := fmt.Sprintf("%s %s %s %s", task.DataSql, where, updSql, orderSql)
|
||||
|
||||
log, err := app.doDataSync(sql, task)
|
||||
if err != nil {
|
||||
@@ -223,6 +234,12 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
|
||||
result := make([]map[string]any, 0)
|
||||
var queryColumns []*dbi.QueryColumn
|
||||
|
||||
// 如果有数据库别名,则从UpdField中去掉数据库别名, 如:a.id => id,用于获取字段具体名称
|
||||
updFieldName := task.UpdField
|
||||
if task.UpdField != "" && strings.Contains(task.UpdField, ".") {
|
||||
updFieldName = strings.Split(task.UpdField, ".")[1]
|
||||
}
|
||||
|
||||
err = srcConn.WalkQueryRows(context.Background(), sql, func(row map[string]any, columns []*dbi.QueryColumn) error {
|
||||
if len(queryColumns) == 0 {
|
||||
queryColumns = columns
|
||||
@@ -230,7 +247,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
|
||||
// 遍历columns 取task.UpdField的字段类型
|
||||
updFieldType = dbi.DataTypeString
|
||||
for _, column := range columns {
|
||||
if strings.EqualFold(strings.ToLower(column.Name), strings.ToLower(task.UpdField)) {
|
||||
if strings.EqualFold(column.Name, updFieldName) {
|
||||
updFieldType = srcDialect.GetDataConverter().GetDataType(column.Type)
|
||||
break
|
||||
}
|
||||
@@ -240,7 +257,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
|
||||
total++
|
||||
result = append(result, row)
|
||||
if total%batchSize == 0 {
|
||||
if err := app.srcData2TargetDb(result, fieldMap, columns, updFieldType, task, srcDialect, targetConn, targetDbTx); err != nil {
|
||||
if err := app.srcData2TargetDb(result, fieldMap, columns, updFieldType, updFieldName, task, srcDialect, targetConn, targetDbTx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -262,15 +279,19 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
|
||||
|
||||
// 处理剩余的数据
|
||||
if len(result) > 0 {
|
||||
if err := app.srcData2TargetDb(result, fieldMap, queryColumns, updFieldType, task, srcDialect, targetConn, targetDbTx); err != nil {
|
||||
if err := app.srcData2TargetDb(result, fieldMap, queryColumns, updFieldType, updFieldName, task, srcDialect, targetConn, targetDbTx); err != nil {
|
||||
targetDbTx.Rollback()
|
||||
return syncLog, err
|
||||
}
|
||||
}
|
||||
|
||||
// 如果是mssql,暂不手动提交事务,否则报错 mssql: The COMMIT TRANSACTION request has no corresponding BEGIN TRANSACTION.
|
||||
if err := targetDbTx.Commit(); err != nil {
|
||||
if targetConn.Info.Type != dbi.DbTypeMssql {
|
||||
return syncLog, errorx.NewBiz("数据同步-目标数据库事务提交失败: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
logx.Infof("同步任务:[%s],执行完毕,保存记录成功:[%d]条", task.TaskName, total)
|
||||
|
||||
// 保存执行成功日志
|
||||
@@ -282,7 +303,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
|
||||
return syncLog, nil
|
||||
}
|
||||
|
||||
func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, columns []*dbi.QueryColumn, updFieldType dbi.DataType, task *entity.DataSyncTask, srcDialect dbi.Dialect, targetDbConn *dbi.DbConn, targetDbTx *sql.Tx) error {
|
||||
func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap []map[string]string, columns []*dbi.QueryColumn, updFieldType dbi.DataType, updFieldName string, task *entity.DataSyncTask, srcDialect dbi.Dialect, targetDbConn *dbi.DbConn, targetDbTx *sql.Tx) error {
|
||||
|
||||
// 遍历src字段列表,取出字段对应的类型
|
||||
var srcColumnTypes = make(map[string]string)
|
||||
@@ -305,9 +326,9 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
|
||||
data = append(data, rowData)
|
||||
}
|
||||
// 解决字段大小写问题
|
||||
updFieldVal := srcRes[len(srcRes)-1][strings.ToUpper(task.UpdField)]
|
||||
updFieldVal := srcRes[len(srcRes)-1][strings.ToUpper(updFieldName)]
|
||||
if updFieldVal == "" || updFieldVal == nil {
|
||||
updFieldVal = srcRes[len(srcRes)-1][strings.ToLower(task.UpdField)]
|
||||
updFieldVal = srcRes[len(srcRes)-1][strings.ToLower(updFieldName)]
|
||||
}
|
||||
|
||||
task.UpdFieldVal = srcDialect.GetDataConverter().FormatData(updFieldVal, updFieldType)
|
||||
@@ -338,7 +359,7 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
|
||||
}
|
||||
|
||||
// 目标数据库执行sql批量插入
|
||||
_, err := targetDbConn.GetDialect().BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values)
|
||||
_, err := targetDbConn.GetDialect().BatchInsert(targetDbTx, task.TargetTableName, targetWrapColumns, values, task.DuplicateStrategy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -61,6 +61,8 @@ func (dbType DbType) RemoveQuote(name string) string {
|
||||
return removeQuote(name, "`")
|
||||
case DbTypePostgres, DbTypeGauss, DbTypeKingbaseEs, DbTypeVastbase:
|
||||
return removeQuote(name, `"`)
|
||||
case DbTypeMssql:
|
||||
return strings.Trim(name, "[]")
|
||||
default:
|
||||
return removeQuote(name, `"`)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,15 @@ const (
|
||||
DataTypeDateTime DataType = "datetime"
|
||||
)
|
||||
|
||||
const (
|
||||
// -1. 无操作
|
||||
DuplicateStrategyNone = -1
|
||||
// 1. 忽略
|
||||
DuplicateStrategyIgnore = 1
|
||||
// 2. 更新
|
||||
DuplicateStrategyUpdate = 2
|
||||
)
|
||||
|
||||
// 数据库服务实例信息
|
||||
type DbServer struct {
|
||||
Version string `json:"version"` // 版本信息
|
||||
@@ -111,7 +120,7 @@ type Dialect interface {
|
||||
GetDbProgram() (DbProgram, error)
|
||||
|
||||
// 批量保存数据
|
||||
BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error)
|
||||
BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error)
|
||||
|
||||
// 获取数据转换器用于解析格式化列数据等
|
||||
GetDataConverter() DataConverter
|
||||
|
||||
@@ -3,7 +3,7 @@ package dbi
|
||||
import "database/sql"
|
||||
|
||||
var (
|
||||
metas map[DbType]Meta = make(map[DbType]Meta)
|
||||
metas = make(map[DbType]Meta)
|
||||
)
|
||||
|
||||
// 注册数据库类型与dbmeta
|
||||
|
||||
@@ -261,35 +261,114 @@ var (
|
||||
dateRegexp = regexp.MustCompile(`(?i)date`)
|
||||
// 时间类型
|
||||
timeRegexp = regexp.MustCompile(`(?i)time`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
func (dd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
func (dd *DMDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
// 执行批量insert sql
|
||||
// insert into "table_name" ("column1", "column2", ...) values (value1, value2, ...)
|
||||
|
||||
dbType := dd.dc.Info.Type
|
||||
|
||||
// 无需处理重复数据,直接执行批量insert
|
||||
if duplicateStrategy == dbi.DuplicateStrategyNone || duplicateStrategy == 0 {
|
||||
return dd.batchInsertSimple(dbType, tx, tableName, columns, values)
|
||||
} else { // 执行MERGE INTO语句
|
||||
return dd.batchInsertMerge(dbType, tx, tableName, columns, values)
|
||||
}
|
||||
}
|
||||
|
||||
func (dd *DMDialect) batchInsertSimple(dbType dbi.DbType, tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
// 生成占位符字符串:如:(?,?)
|
||||
// 重复字符串并用逗号连接
|
||||
repeated := strings.Repeat("?,", len(columns))
|
||||
// 去除最后一个逗号,占位符由括号包裹
|
||||
placeholder := fmt.Sprintf("(%s)", strings.TrimSuffix(repeated, ","))
|
||||
|
||||
sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", dd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
sqlTemp := fmt.Sprintf("insert into %s (%s) values %s", dbType.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
effRows := 0
|
||||
for _, value := range values {
|
||||
// 达梦数据库只能一条条的执行insert
|
||||
er, err := dd.dc.TxExec(tx, sqlTemp, value...)
|
||||
res, err := dd.dc.TxExec(tx, sqlTemp, value...)
|
||||
if err != nil {
|
||||
logx.Errorf("执行sql失败:%s", err.Error())
|
||||
return int64(effRows), err
|
||||
logx.Errorf("执行sql失败:%s, sql: [ %s ]", err.Error(), sqlTemp)
|
||||
}
|
||||
effRows += int(er)
|
||||
effRows += int(res)
|
||||
}
|
||||
// 执行批量insert sql
|
||||
return int64(effRows), nil
|
||||
}
|
||||
|
||||
func (dd *DMDialect) batchInsertMerge(dbType dbi.DbType, tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
// 查询主键字段
|
||||
uniqueCols := make([]string, 0)
|
||||
caseSqls := make([]string, 0)
|
||||
tableCols, _ := dd.GetColumns(tableName)
|
||||
identityCols := make([]string, 0)
|
||||
for _, col := range tableCols {
|
||||
if col.IsPrimaryKey {
|
||||
uniqueCols = append(uniqueCols, col.ColumnName)
|
||||
caseSqls = append(caseSqls, fmt.Sprintf("( T1.%s = T2.%s )", dbType.QuoteIdentifier(col.ColumnName), dbType.QuoteIdentifier(col.ColumnName)))
|
||||
}
|
||||
if col.IsIdentity {
|
||||
// 自增字段不放入insert内,即使是设置了identity_insert on也不起作用
|
||||
identityCols = append(identityCols, dbType.QuoteIdentifier(col.ColumnName))
|
||||
}
|
||||
}
|
||||
// 查询唯一索引涉及到的字段,并组装到match条件内
|
||||
indexs, _ := dd.GetTableIndex(tableName)
|
||||
if indexs != nil {
|
||||
for _, index := range indexs {
|
||||
if index.IsUnique {
|
||||
cols := strings.Split(index.ColumnName, ",")
|
||||
tmp := make([]string, 0)
|
||||
for _, col := range cols {
|
||||
uniqueCols = append(uniqueCols, col)
|
||||
tmp = append(tmp, fmt.Sprintf(" T1.%s = T2.%s ", dbType.QuoteIdentifier(col), dbType.QuoteIdentifier(col)))
|
||||
}
|
||||
caseSqls = append(caseSqls, fmt.Sprintf("( %s )", strings.Join(tmp, " AND ")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 重复数据处理策略
|
||||
phs := make([]string, 0)
|
||||
insertVals := make([]string, 0)
|
||||
upds := make([]string, 0)
|
||||
insertCols := make([]string, 0)
|
||||
for _, column := range columns {
|
||||
phs = append(phs, fmt.Sprintf("? %s", column))
|
||||
if !collx.ArrayContains(uniqueCols, dbType.RemoveQuote(column)) {
|
||||
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
|
||||
}
|
||||
if !collx.ArrayContains(identityCols, column) {
|
||||
insertCols = append(insertCols, fmt.Sprintf("%s", column))
|
||||
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
|
||||
}
|
||||
|
||||
}
|
||||
t2s := make([]string, 0)
|
||||
for i := 0; i < len(values); i++ {
|
||||
t2s = append(t2s, fmt.Sprintf("SELECT %s FROM dual", strings.Join(phs, ",")))
|
||||
}
|
||||
t2 := strings.Join(t2s, " UNION ALL ")
|
||||
|
||||
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON " + strings.Join(caseSqls, " OR ")
|
||||
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ")"
|
||||
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
|
||||
|
||||
// 把二维数组转为一维数组
|
||||
var args []any
|
||||
for _, v := range values {
|
||||
args = append(args, v...)
|
||||
}
|
||||
|
||||
return dd.dc.TxExec(tx, sqlTemp, args...)
|
||||
}
|
||||
|
||||
func (dd *DMDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
return converter
|
||||
}
|
||||
|
||||
type DataConverter struct {
|
||||
@@ -328,6 +407,22 @@ func (dd *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
|
||||
}
|
||||
|
||||
func (dd *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
|
||||
// 如果dataType是datetime而dbColumnValue是string类型,则需要转换为time.Time类型
|
||||
_, ok := dbColumnValue.(string)
|
||||
if ok {
|
||||
if dataType == dbi.DataTypeDateTime {
|
||||
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeDate {
|
||||
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeTime {
|
||||
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
}
|
||||
return dbColumnValue
|
||||
}
|
||||
|
||||
|
||||
@@ -135,13 +135,11 @@ func (md *MssqlDialect) GetPrimaryKey(tablename string) (string, error) {
|
||||
return columns[0].ColumnName, nil
|
||||
}
|
||||
|
||||
// 获取表索引信息
|
||||
func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
func (md *MssqlDialect) getTableIndexWithPK(tableName string) ([]dbi.Index, error) {
|
||||
_, res, err := md.dc.Query(dbi.GetLocalSql(MSSQL_META_FILE, MSSQL_INDEX_INFO_KEY), md.currentSchema(), tableName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indexs := make([]dbi.Index, 0)
|
||||
for _, re := range res {
|
||||
indexs = append(indexs, dbi.Index{
|
||||
@@ -153,18 +151,14 @@ func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
SeqInIndex: anyx.ConvInt(re["seqInIndex"]),
|
||||
})
|
||||
}
|
||||
// 把查询结果以索引名分组,索引字段以逗号连接
|
||||
// 把查询结果以索引名分组,多个索引字段以逗号连接
|
||||
result := make([]dbi.Index, 0)
|
||||
key := ""
|
||||
for _, v := range indexs {
|
||||
// 当前的索引名
|
||||
in := v.IndexName
|
||||
// 过滤掉主键索引,主键索引名为PK__开头的
|
||||
if strings.HasPrefix(in, "PK__") {
|
||||
continue
|
||||
}
|
||||
if key == in {
|
||||
// 索引字段已根据名称和顺序排序,故取最后一个即可
|
||||
// 索引字段已根据名称和字段顺序排序,故取最后一个即可
|
||||
i := len(result) - 1
|
||||
// 同索引字段以逗号连接
|
||||
result[i].ColumnName = result[i].ColumnName + "," + v.ColumnName
|
||||
@@ -173,6 +167,20 @@ func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
result = append(result, v)
|
||||
}
|
||||
}
|
||||
return indexs, nil
|
||||
}
|
||||
|
||||
// 获取表索引信息
|
||||
func (md *MssqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
indexs, _ := md.getTableIndexWithPK(tableName)
|
||||
result := make([]dbi.Index, 0)
|
||||
// 过滤掉主键索引,主键索引名为PK__开头的
|
||||
for _, v := range indexs {
|
||||
in := v.IndexName
|
||||
if strings.HasPrefix(in, "PK__") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -289,8 +297,39 @@ func (md *MssqlDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", md.dc.Info.Type)
|
||||
}
|
||||
|
||||
func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
|
||||
if duplicateStrategy == dbi.DuplicateStrategyUpdate {
|
||||
return md.batchInsertMerge(tx, tableName, columns, values, duplicateStrategy)
|
||||
}
|
||||
|
||||
return md.batchInsertSimple(tx, tableName, columns, values, duplicateStrategy)
|
||||
}
|
||||
|
||||
func (md *MssqlDialect) batchInsertSimple(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
schema := md.currentSchema()
|
||||
ignoreDupSql := ""
|
||||
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
|
||||
// ALTER TABLE dbo.TEST ADD CONSTRAINT uniqueRows UNIQUE (ColA, ColB, ColC, ColD) WITH (IGNORE_DUP_KEY = ON)
|
||||
indexs, _ := md.getTableIndexWithPK(tableName)
|
||||
// 收集唯一索引涉及到的字段
|
||||
uniqueColumns := make([]string, 0)
|
||||
for _, index := range indexs {
|
||||
if index.IsUnique {
|
||||
cols := strings.Split(index.ColumnName, ",")
|
||||
for _, col := range cols {
|
||||
if !collx.ArrayContains(uniqueColumns, col) {
|
||||
uniqueColumns = append(uniqueColumns, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(uniqueColumns) > 0 {
|
||||
// 设置忽略重复键
|
||||
ignoreDupSql = fmt.Sprintf("ALTER TABLE %s.%s ADD CONSTRAINT uniqueRows UNIQUE (%s) WITH (IGNORE_DUP_KEY = {sign})", schema, tableName, strings.Join(uniqueColumns, ","))
|
||||
_, _ = md.dc.TxExec(tx, strings.ReplaceAll(ignoreDupSql, "{sign}", "ON"))
|
||||
}
|
||||
}
|
||||
|
||||
// 生成占位符字符串:如:(?,?)
|
||||
// 重复字符串并用逗号连接
|
||||
@@ -312,18 +351,86 @@ func (md *MssqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
|
||||
for _, v := range values {
|
||||
args = append(args, v...)
|
||||
}
|
||||
|
||||
// 设置允许填充自增列之后,显示指定列名可以插入自增列
|
||||
_, _ = md.dc.Exec(fmt.Sprintf("set identity_insert \"%s\" on", tableName))
|
||||
identityInsertOn := fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, tableName)
|
||||
|
||||
exec, err := md.dc.TxExec(tx, sqlStr, args...)
|
||||
res, err := md.dc.TxExec(tx, fmt.Sprintf("%s %s", identityInsertOn, sqlStr), args...)
|
||||
|
||||
_, _ = md.dc.Exec(fmt.Sprintf("set identity_insert \"%s\" off", tableName))
|
||||
// 执行完之后,设置忽略重复键
|
||||
if ignoreDupSql != "" {
|
||||
_, _ = md.dc.TxExec(tx, strings.ReplaceAll(ignoreDupSql, "{sign}", "OFF"))
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
return exec, err
|
||||
func (md *MssqlDialect) batchInsertMerge(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
schema := md.currentSchema()
|
||||
dbType := md.dc.Info.Type
|
||||
|
||||
// 收集MERGE 语句的 ON 子句条件
|
||||
caseSqls := make([]string, 0)
|
||||
pkCols := make([]string, 0)
|
||||
|
||||
// 查询取出自增列字段, merge update不能修改自增列
|
||||
identityCols := make([]string, 0)
|
||||
cols, err := md.GetColumns(tableName)
|
||||
for _, col := range cols {
|
||||
if col.IsIdentity {
|
||||
identityCols = append(identityCols, col.ColumnName)
|
||||
}
|
||||
if col.IsPrimaryKey {
|
||||
pkCols = append(pkCols, col.ColumnName)
|
||||
name := dbType.QuoteIdentifier(col.ColumnName)
|
||||
caseSqls = append(caseSqls, fmt.Sprintf(" T1.%s = T2.%s ", name, name))
|
||||
}
|
||||
}
|
||||
if len(pkCols) == 0 {
|
||||
return md.batchInsertSimple(tx, tableName, columns, values, duplicateStrategy)
|
||||
}
|
||||
// 重复数据处理策略
|
||||
insertVals := make([]string, 0)
|
||||
upds := make([]string, 0)
|
||||
insertCols := make([]string, 0)
|
||||
// 源数据占位sql
|
||||
phs := make([]string, 0)
|
||||
for _, column := range columns {
|
||||
if !collx.ArrayContains(identityCols, dbType.RemoveQuote(column)) {
|
||||
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
|
||||
}
|
||||
insertCols = append(insertCols, fmt.Sprintf("%s", column))
|
||||
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
|
||||
phs = append(phs, fmt.Sprintf("? %s", column))
|
||||
}
|
||||
|
||||
// 把二维数组转为一维数组
|
||||
var args []any
|
||||
tmp := fmt.Sprintf("select %s", strings.Join(phs, ","))
|
||||
t2s := make([]string, 0)
|
||||
for _, v := range values {
|
||||
args = append(args, v...)
|
||||
t2s = append(t2s, tmp)
|
||||
}
|
||||
t2 := strings.Join(t2s, " UNION ALL ")
|
||||
|
||||
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(schema) + "." + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON " + strings.Join(caseSqls, " AND ")
|
||||
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ") "
|
||||
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
|
||||
|
||||
// 设置允许填充自增列之后,显示指定列名可以插入自增列
|
||||
identityInsertOn := fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, tableName)
|
||||
|
||||
// 执行merge sql,必须要以分号结尾
|
||||
res, err := md.dc.TxExec(tx, fmt.Sprintf("%s %s;", identityInsertOn, sqlTemp), args...)
|
||||
|
||||
if err != nil {
|
||||
logx.Errorf("执行sql失败:%s, sql: [ %s ]", err.Error(), sqlTemp)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (md *MssqlDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
return converter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -335,6 +442,8 @@ var (
|
||||
dateRegexp = regexp.MustCompile(`(?i)date`)
|
||||
// 时间类型
|
||||
timeRegexp = regexp.MustCompile(`(?i)time`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
type DataConverter struct {
|
||||
@@ -364,6 +473,20 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
|
||||
}
|
||||
|
||||
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
|
||||
// 如果dataType是datetime而dbColumnValue是string类型,则需要转换为time.Time类型
|
||||
_, ok := dbColumnValue.(string)
|
||||
if dataType == dbi.DataTypeDateTime && ok {
|
||||
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeDate && ok {
|
||||
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeTime && ok {
|
||||
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
return dbColumnValue
|
||||
}
|
||||
|
||||
@@ -409,12 +532,10 @@ func (md *MssqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
// 复制数据
|
||||
// 设置允许填充自增列之后,显示指定列名可以插入自增列
|
||||
identityInsertOn := ""
|
||||
identityInsertOff := ""
|
||||
if hasIdentity {
|
||||
identityInsertOn = fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] ON", schema, newTableName)
|
||||
identityInsertOff = fmt.Sprintf("SET IDENTITY_INSERT [%s].[%s] OFF", schema, newTableName)
|
||||
}
|
||||
_, err = md.dc.Exec(fmt.Sprintf(" %s INSERT INTO [%s].[%s] (%s) SELECT * FROM [%s].[%s] %s", identityInsertOn, schema, newTableName, columnsSql, schema, copy.TableName, identityInsertOff))
|
||||
_, err = md.dc.Exec(fmt.Sprintf(" %s INSERT INTO [%s].[%s] (%s) SELECT * FROM [%s].[%s]", identityInsertOn, schema, newTableName, columnsSql, schema, copy.TableName))
|
||||
if err != nil {
|
||||
logx.Warnf("复制表[%s]数据失败: %s", copy.TableName, err.Error())
|
||||
}
|
||||
|
||||
@@ -10,14 +10,14 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
meta := new(Meta)
|
||||
meta := new(MssqlMeta)
|
||||
dbi.Register(dbi.DbTypeMssql, meta)
|
||||
}
|
||||
|
||||
type Meta struct {
|
||||
type MssqlMeta struct {
|
||||
}
|
||||
|
||||
func (md *Meta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
|
||||
func (md *MssqlMeta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
|
||||
err := d.IfUseSshTunnelChangeIpPort()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -52,6 +52,6 @@ func (md *Meta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
|
||||
return sql.Open(driverName, dsn)
|
||||
}
|
||||
|
||||
func (md *Meta) GetDialect(conn *dbi.DbConn) dbi.Dialect {
|
||||
func (md *MssqlMeta) GetDialect(conn *dbi.DbConn) dbi.Dialect {
|
||||
return &MssqlDialect{conn}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,6 @@ func (md *MysqlDialect) GetDbNames() ([]string, error) {
|
||||
for _, re := range res {
|
||||
databases = append(databases, anyx.ConvString(re["dbname"]))
|
||||
}
|
||||
|
||||
return databases, nil
|
||||
}
|
||||
|
||||
@@ -173,7 +172,7 @@ func (md *MysqlDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return NewDbProgramMysql(md.dc), nil
|
||||
}
|
||||
|
||||
func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
// 生成占位符字符串:如:(?,?)
|
||||
// 重复字符串并用逗号连接
|
||||
repeated := strings.Repeat("?,", len(columns))
|
||||
@@ -188,7 +187,14 @@ func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
|
||||
// 去除最后一个逗号
|
||||
placeholder = strings.TrimSuffix(repeated, ",")
|
||||
|
||||
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
prefix := "insert into"
|
||||
if duplicateStrategy == 1 {
|
||||
prefix = "insert ignore into"
|
||||
} else if duplicateStrategy == 2 {
|
||||
prefix = "replace into"
|
||||
}
|
||||
|
||||
sqlStr := fmt.Sprintf("%s %s (%s) values %s", prefix, md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
// 执行批量insert sql
|
||||
// 把二维数组转为一维数组
|
||||
var args []any
|
||||
@@ -199,7 +205,7 @@ func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
|
||||
}
|
||||
|
||||
func (md *MysqlDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
return converter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -211,6 +217,8 @@ var (
|
||||
dateRegexp = regexp.MustCompile(`(?i)date`)
|
||||
// 时间类型
|
||||
timeRegexp = regexp.MustCompile(`(?i)time`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
type DataConverter struct {
|
||||
@@ -240,6 +248,22 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
|
||||
}
|
||||
|
||||
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
|
||||
// 如果dataType是datetime而dbColumnValue是string类型,则需要转换为time.Time类型
|
||||
_, ok := dbColumnValue.(string)
|
||||
if ok {
|
||||
if dataType == dbi.DataTypeDateTime {
|
||||
res, _ := time.Parse(time.DateTime, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeDate {
|
||||
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeTime {
|
||||
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
}
|
||||
return dbColumnValue
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"mayfly-go/internal/db/dbm/dbi"
|
||||
"mayfly-go/pkg/errorx"
|
||||
"mayfly-go/pkg/logx"
|
||||
"mayfly-go/pkg/utils/anyx"
|
||||
"mayfly-go/pkg/utils/collx"
|
||||
"regexp"
|
||||
@@ -252,13 +253,7 @@ func (od *OracleDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", od.dc.Info.Type)
|
||||
}
|
||||
|
||||
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
//INSERT ALL
|
||||
//INTO my_table(field_1,field_2) VALUES (value_1,value_2)
|
||||
//INTO my_table(field_1,field_2) VALUES (value_3,value_4)
|
||||
//INTO my_table(field_1,field_2) VALUES (value_5,value_6)
|
||||
//SELECT 1 FROM DUAL;
|
||||
|
||||
func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
if len(values) <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -269,27 +264,122 @@ func (od *OracleDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
|
||||
args = append(args, v...)
|
||||
}
|
||||
|
||||
// 拼接oracle批量插入语句
|
||||
sqlArr := make([]string, 0)
|
||||
sqlArr = append(sqlArr, "INSERT ALL")
|
||||
if duplicateStrategy == dbi.DuplicateStrategyNone || duplicateStrategy == 0 || duplicateStrategy == dbi.DuplicateStrategyIgnore {
|
||||
return od.batchInsertSimple(od.dc.Info.Type, tableName, columns, values, duplicateStrategy, tx)
|
||||
} else {
|
||||
return od.batchInsertMergeSql(od.dc.Info.Type, tableName, columns, values, args, tx)
|
||||
}
|
||||
}
|
||||
|
||||
// 简单批量插入sql,无需判断键冲突策略
|
||||
func (od *OracleDialect) batchInsertSimple(dbType dbi.DbType, tableName string, columns []string, values [][]any, duplicateStrategy int, tx *sql.Tx) (int64, error) {
|
||||
|
||||
// 忽略键冲突策略
|
||||
ignore := ""
|
||||
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
|
||||
// 查出唯一索引涉及的字段
|
||||
indexs, _ := od.GetTableIndex(tableName)
|
||||
if indexs != nil {
|
||||
arr := make([]string, 0)
|
||||
for _, index := range indexs {
|
||||
if index.IsUnique {
|
||||
cols := strings.Split(index.ColumnName, ",")
|
||||
for _, col := range cols {
|
||||
if !collx.ArrayContains(arr, col) {
|
||||
arr = append(arr, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ignore = fmt.Sprintf("/*+ IGNORE_ROW_ON_DUPKEY_INDEX(%s(%s)) */", tableName, strings.Join(arr, ","))
|
||||
}
|
||||
}
|
||||
effRows := 0
|
||||
for _, value := range values {
|
||||
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
|
||||
var placeholder []string
|
||||
for i := 0; i < len(value); i++ {
|
||||
placeholder = append(placeholder, fmt.Sprintf(":%d", i+1))
|
||||
}
|
||||
sqlTemp := fmt.Sprintf("INSERT %s INTO %s (%s) VALUES (%s)", ignore, dbType.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ","))
|
||||
|
||||
// oracle数据库为了兼容ignore主键冲突,只能一条条的执行insert
|
||||
res, err := od.dc.TxExec(tx, sqlTemp, value...)
|
||||
if err != nil {
|
||||
logx.Errorf("执行sql失败:%s, sql: [ %s ]", err.Error(), sqlTemp)
|
||||
}
|
||||
effRows += int(res)
|
||||
}
|
||||
return int64(effRows), nil
|
||||
}
|
||||
|
||||
func (od *OracleDialect) batchInsertMergeSql(dbType dbi.DbType, tableName string, columns []string, values [][]any, args []any, tx *sql.Tx) (int64, error) {
|
||||
// 查询主键字段
|
||||
uniqueCols := make([]string, 0)
|
||||
caseSqls := make([]string, 0)
|
||||
// 查询唯一索引涉及到的字段,并组装到match条件内
|
||||
indexs, _ := od.GetTableIndex(tableName)
|
||||
if indexs != nil {
|
||||
for _, index := range indexs {
|
||||
if index.IsUnique {
|
||||
cols := strings.Split(index.ColumnName, ",")
|
||||
tmp := make([]string, 0)
|
||||
for _, col := range cols {
|
||||
if !collx.ArrayContains(uniqueCols, col) {
|
||||
uniqueCols = append(uniqueCols, col)
|
||||
}
|
||||
tmp = append(tmp, fmt.Sprintf(" T1.%s = T2.%s ", dbType.QuoteIdentifier(col), dbType.QuoteIdentifier(col)))
|
||||
}
|
||||
caseSqls = append(caseSqls, fmt.Sprintf("( %s )", strings.Join(tmp, " AND ")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果caseSqls为空,则说明没有唯一键,直接使用简单批量插入
|
||||
if len(caseSqls) == 0 {
|
||||
return od.batchInsertSimple(dbType, tableName, columns, values, dbi.DuplicateStrategyNone, tx)
|
||||
}
|
||||
|
||||
// 重复数据处理策略
|
||||
insertVals := make([]string, 0)
|
||||
upds := make([]string, 0)
|
||||
insertCols := make([]string, 0)
|
||||
for _, column := range columns {
|
||||
if !collx.ArrayContains(uniqueCols, dbType.RemoveQuote(column)) {
|
||||
upds = append(upds, fmt.Sprintf("T1.%s = T2.%s", column, column))
|
||||
}
|
||||
insertCols = append(insertCols, fmt.Sprintf("T1.%s", column))
|
||||
insertVals = append(insertVals, fmt.Sprintf("T2.%s", column))
|
||||
}
|
||||
|
||||
// 生成源数据占位sql
|
||||
t2s := make([]string, 0)
|
||||
// 拼接带占位符的sql oracle的占位符是:1,:2,:3....
|
||||
for i := 0; i < len(args); i += len(columns) {
|
||||
var placeholder []string
|
||||
for j := 0; j < len(columns); j++ {
|
||||
placeholder = append(placeholder, fmt.Sprintf(":%d", i+j+1))
|
||||
col := columns[j]
|
||||
placeholder = append(placeholder, fmt.Sprintf(":%d %s", i+j+1, col))
|
||||
}
|
||||
sqlArr = append(sqlArr, fmt.Sprintf("INTO %s (%s) VALUES (%s)", od.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholder, ",")))
|
||||
t2s = append(t2s, fmt.Sprintf("SELECT %s FROM dual", strings.Join(placeholder, ", ")))
|
||||
}
|
||||
sqlArr = append(sqlArr, "SELECT 1 FROM DUAL")
|
||||
|
||||
t2 := strings.Join(t2s, " UNION ALL ")
|
||||
|
||||
sqlTemp := "MERGE INTO " + dbType.QuoteIdentifier(tableName) + " T1 USING (" + t2 + ") T2 ON (" + strings.Join(caseSqls, " OR ") + ") "
|
||||
sqlTemp += "WHEN NOT MATCHED THEN INSERT (" + strings.Join(insertCols, ",") + ") VALUES (" + strings.Join(insertVals, ",") + ") "
|
||||
sqlTemp += "WHEN MATCHED THEN UPDATE SET " + strings.Join(upds, ",")
|
||||
|
||||
// 执行批量insert sql
|
||||
res, err := od.dc.TxExec(tx, strings.Join(sqlArr, " "), args...)
|
||||
res, err := od.dc.TxExec(tx, sqlTemp, args...)
|
||||
if err != nil {
|
||||
logx.Errorf("执行sql失败:%s, sql: [ %s ]", err.Error(), sqlTemp)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (od *OracleDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
return converter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -297,6 +387,8 @@ var (
|
||||
numberTypeRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
|
||||
// 日期时间类型
|
||||
datetimeTypeRegexp = regexp.MustCompile(`(?i)date|timestamp`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
type DataConverter struct {
|
||||
|
||||
@@ -50,7 +50,7 @@ func (md *OraMeta) GetSqlDb(d *dbi.DbInfo) (*sql.DB, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
urlOptions["TIMEOUT"] = "10"
|
||||
urlOptions["TIMEOUT"] = "1000"
|
||||
connStr := go_ora.BuildUrl(d.Host, d.Port, d.Sid, d.Username, d.Password, urlOptions)
|
||||
conn, err := sql.Open(driverName, connStr)
|
||||
if err != nil {
|
||||
|
||||
@@ -25,8 +25,8 @@ type PgsqlDialect struct {
|
||||
dc *dbi.DbConn
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
|
||||
_, res, err := md.dc.Query("SELECT version() as server_version")
|
||||
func (pd *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
|
||||
_, res, err := pd.dc.Query("SELECT version() as server_version")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -36,8 +36,8 @@ func (md *PgsqlDialect) GetDbServer() (*dbi.DbServer, error) {
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) GetDbNames() ([]string, error) {
|
||||
_, res, err := md.dc.Query("SELECT datname AS dbname FROM pg_database WHERE datistemplate = false AND has_database_privilege(datname, 'CONNECT')")
|
||||
func (pd *PgsqlDialect) GetDbNames() ([]string, error) {
|
||||
_, res, err := pd.dc.Query("SELECT datname AS dbname FROM pg_database WHERE datistemplate = false AND has_database_privilege(datname, 'CONNECT')")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -51,8 +51,8 @@ func (md *PgsqlDialect) GetDbNames() ([]string, error) {
|
||||
}
|
||||
|
||||
// 获取表基础元信息, 如表名等
|
||||
func (md *PgsqlDialect) GetTables() ([]dbi.Table, error) {
|
||||
_, res, err := md.dc.Query(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_INFO_KEY))
|
||||
func (pd *PgsqlDialect) GetTables() ([]dbi.Table, error) {
|
||||
_, res, err := pd.dc.Query(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_INFO_KEY))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -72,13 +72,13 @@ func (md *PgsqlDialect) GetTables() ([]dbi.Table, error) {
|
||||
}
|
||||
|
||||
// 获取列元信息, 如列名等
|
||||
func (md *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
|
||||
dbType := md.dc.Info.Type
|
||||
func (pd *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
|
||||
dbType := pd.dc.Info.Type
|
||||
tableName := strings.Join(collx.ArrayMap[string, string](tableNames, func(val string) string {
|
||||
return fmt.Sprintf("'%s'", dbType.RemoveQuote(val))
|
||||
}), ",")
|
||||
|
||||
_, res, err := md.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_COLUMN_MA_KEY), tableName))
|
||||
_, res, err := pd.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_COLUMN_MA_KEY), tableName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -100,8 +100,8 @@ func (md *PgsqlDialect) GetColumns(tableNames ...string) ([]dbi.Column, error) {
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
|
||||
columns, err := md.GetColumns(tablename)
|
||||
func (pd *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
|
||||
columns, err := pd.GetColumns(tablename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -118,8 +118,8 @@ func (md *PgsqlDialect) GetPrimaryKey(tablename string) (string, error) {
|
||||
}
|
||||
|
||||
// 获取表索引信息
|
||||
func (md *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
_, res, err := md.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_INDEX_INFO_KEY), tableName))
|
||||
func (pd *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
_, res, err := pd.dc.Query(fmt.Sprintf(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_INDEX_INFO_KEY), tableName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -155,17 +155,14 @@ func (md *PgsqlDialect) GetTableIndex(tableName string) ([]dbi.Index, error) {
|
||||
}
|
||||
|
||||
// 获取建表ddl
|
||||
func (md *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
|
||||
_, err := md.dc.Exec(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_DDL_KEY))
|
||||
func (pd *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
|
||||
_, err := pd.dc.Exec(dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_TABLE_DDL_KEY))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
_, schemaRes, _ := md.dc.Query("select current_schema() as schema")
|
||||
schemaName := schemaRes[0]["schema"].(string)
|
||||
|
||||
ddlSql := fmt.Sprintf("select showcreatetable('%s','%s') as sql", schemaName, tableName)
|
||||
_, res, err := md.dc.Query(ddlSql)
|
||||
ddlSql := fmt.Sprintf("select showcreatetable('%s','%s') as sql", pd.currentSchema(), tableName)
|
||||
_, res, err := pd.dc.Query(ddlSql)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -174,9 +171,9 @@ func (md *PgsqlDialect) GetTableDDL(tableName string) (string, error) {
|
||||
}
|
||||
|
||||
// 获取pgsql当前连接的库可访问的schemaNames
|
||||
func (md *PgsqlDialect) GetSchemas() ([]string, error) {
|
||||
func (pd *PgsqlDialect) GetSchemas() ([]string, error) {
|
||||
sql := dbi.GetLocalSql(PGSQL_META_FILE, PGSQL_DB_SCHEMAS)
|
||||
_, res, err := md.dc.Query(sql)
|
||||
_, res, err := pd.dc.Query(sql)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -188,11 +185,11 @@ func (md *PgsqlDialect) GetSchemas() ([]string, error) {
|
||||
}
|
||||
|
||||
// GetDbProgram 获取数据库程序模块,用于数据库备份与恢复
|
||||
func (md *PgsqlDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", md.dc.Info.Type)
|
||||
func (pd *PgsqlDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", pd.dc.Info.Type)
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
func (pd *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
// 执行批量insert sql,跟mysql一样 pg或高斯支持批量insert语法
|
||||
// insert into table_name (column1, column2, ...) values (value1, value2, ...), (value1, value2, ...), ...
|
||||
|
||||
@@ -212,14 +209,99 @@ func (md *PgsqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
|
||||
placeholders = append(placeholders, "("+strings.Join(placeholder, ", ")+")")
|
||||
}
|
||||
|
||||
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", md.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholders, ", "))
|
||||
// 执行批量insert sql
|
||||
|
||||
return md.dc.TxExec(tx, sqlStr, args...)
|
||||
// 根据冲突策略生成后缀
|
||||
suffix := ""
|
||||
if pd.dc.Info.Type == dbi.DbTypeGauss {
|
||||
// 高斯db使用ON DUPLICATE KEY UPDATE 语法参考 https://support.huaweicloud.com/distributed-devg-v3-gaussdb/gaussdb-12-0607.html#ZH-CN_TOPIC_0000001633948138
|
||||
suffix = pd.gaussOnDuplicateStrategySql(duplicateStrategy, tableName, columns)
|
||||
} else {
|
||||
// pgsql 默认使用 on conflict 语法参考 http://www.postgres.cn/docs/12/sql-insert.html
|
||||
// vastbase语法参考 https://docs.vastdata.com.cn/zh/docs/VastbaseE100Ver3.0.0/doc/SQL%E8%AF%AD%E6%B3%95/INSERT.html
|
||||
// kingbase语法参考 https://help.kingbase.com.cn/v8/development/sql-plsql/sql/SQL_Statements_9.html#insert
|
||||
suffix = pd.pgsqlOnDuplicateStrategySql(duplicateStrategy, tableName, columns)
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
sqlStr := fmt.Sprintf("insert into %s (%s) values %s %s", pd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), strings.Join(placeholders, ", "), suffix)
|
||||
// 执行批量insert sql
|
||||
|
||||
return pd.dc.TxExec(tx, sqlStr, args...)
|
||||
}
|
||||
|
||||
// pgsql默认唯一键冲突策略
|
||||
func (pd PgsqlDialect) pgsqlOnDuplicateStrategySql(duplicateStrategy int, tableName string, columns []string) string {
|
||||
suffix := ""
|
||||
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
|
||||
suffix = " \n on conflict do nothing"
|
||||
} else if duplicateStrategy == dbi.DuplicateStrategyUpdate {
|
||||
// 生成 on conflict () do update set column1 = excluded.column1, column2 = excluded.column2, ...
|
||||
var updateColumns []string
|
||||
for _, col := range columns {
|
||||
updateColumns = append(updateColumns, fmt.Sprintf("%s = excluded.%s", col, col))
|
||||
}
|
||||
// 查询唯一键名,拼接冲突sql
|
||||
_, keyRes, _ := pd.dc.Query("SELECT constraint_name FROM information_schema.table_constraints WHERE constraint_schema = $1 AND table_name = $2 AND constraint_type in ('PRIMARY KEY', 'UNIQUE') ", pd.currentSchema(), tableName)
|
||||
if len(keyRes) > 0 {
|
||||
for _, re := range keyRes {
|
||||
key := anyx.ToString(re["constraint_name"])
|
||||
if key != "" {
|
||||
suffix += fmt.Sprintf(" \n on conflict on constraint %s do update set %s \n", key, strings.Join(updateColumns, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return suffix
|
||||
}
|
||||
|
||||
// 高斯db唯一键冲突策略,使用ON DUPLICATE KEY UPDATE 参考:https://support.huaweicloud.com/distributed-devg-v3-gaussdb/gaussdb-12-0607.html#ZH-CN_TOPIC_0000001633948138
|
||||
func (pd PgsqlDialect) gaussOnDuplicateStrategySql(duplicateStrategy int, tableName string, columns []string) string {
|
||||
suffix := ""
|
||||
if duplicateStrategy == dbi.DuplicateStrategyIgnore {
|
||||
suffix = " \n ON DUPLICATE KEY UPDATE NOTHING"
|
||||
} else if duplicateStrategy == dbi.DuplicateStrategyUpdate {
|
||||
|
||||
// 查出表里的唯一键涉及的字段
|
||||
var uniqueColumns []string
|
||||
indexs, err := pd.GetTableIndex(tableName)
|
||||
if err == nil {
|
||||
for _, index := range indexs {
|
||||
if index.IsUnique {
|
||||
cols := strings.Split(index.ColumnName, ",")
|
||||
for _, col := range cols {
|
||||
if !collx.ArrayContains(uniqueColumns, strings.ToLower(col)) {
|
||||
uniqueColumns = append(uniqueColumns, strings.ToLower(col))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suffix = " \n ON DUPLICATE KEY UPDATE "
|
||||
for i, col := range columns {
|
||||
// ON DUPLICATE KEY UPDATE语句不支持更新唯一键字段,所以得去掉
|
||||
if !collx.ArrayContains(uniqueColumns, pd.dc.Info.Type.RemoveQuote(strings.ToLower(col))) {
|
||||
suffix += fmt.Sprintf("%s = excluded.%s", col, col)
|
||||
if i < len(columns)-1 {
|
||||
suffix += ", "
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return suffix
|
||||
}
|
||||
|
||||
// 从连接信息中获取数据库和schema信息
|
||||
func (pd *PgsqlDialect) currentSchema() string {
|
||||
dbName := pd.dc.Info.Database
|
||||
schema := ""
|
||||
arr := strings.Split(dbName, "/")
|
||||
if len(arr) == 2 {
|
||||
schema = arr[1]
|
||||
}
|
||||
return schema
|
||||
}
|
||||
|
||||
func (pd *PgsqlDialect) GetDataConverter() dbi.DataConverter {
|
||||
return converter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -231,6 +313,8 @@ var (
|
||||
dateRegexp = regexp.MustCompile(`(?i)date`)
|
||||
// 时间类型
|
||||
timeRegexp = regexp.MustCompile(`(?i)time`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
type DataConverter struct {
|
||||
@@ -272,19 +356,33 @@ func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) st
|
||||
}
|
||||
|
||||
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
|
||||
// 如果dataType是datetime而dbColumnValue是string类型,则需要转换为time.Time类型
|
||||
_, ok := dbColumnValue.(string)
|
||||
if dataType == dbi.DataTypeDateTime && ok {
|
||||
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeDate && ok {
|
||||
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
if dataType == dbi.DataTypeTime && ok {
|
||||
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
|
||||
return res
|
||||
}
|
||||
return dbColumnValue
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) IsGauss() bool {
|
||||
return strings.Contains(md.dc.Info.Params, "gauss")
|
||||
func (pd *PgsqlDialect) IsGauss() bool {
|
||||
return strings.Contains(pd.dc.Info.Params, "gauss")
|
||||
}
|
||||
|
||||
func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
func (pd *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
tableName := copy.TableName
|
||||
// 生成新表名,为老表明+_copy_时间戳
|
||||
newTableName := tableName + "_copy_" + time.Now().Format("20060102150405")
|
||||
// 执行根据旧表创建新表
|
||||
_, err := md.dc.Exec(fmt.Sprintf("create table %s (like %s)", newTableName, tableName))
|
||||
_, err := pd.dc.Exec(fmt.Sprintf("create table %s (like %s)", newTableName, tableName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -292,12 +390,12 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
// 复制数据
|
||||
if copy.CopyData {
|
||||
go func() {
|
||||
_, _ = md.dc.Exec(fmt.Sprintf("insert into %s select * from %s", newTableName, tableName))
|
||||
_, _ = pd.dc.Exec(fmt.Sprintf("insert into %s select * from %s", newTableName, tableName))
|
||||
}()
|
||||
}
|
||||
|
||||
// 查询旧表的自增字段名 重新设置新表的序列序列器
|
||||
_, res, err := md.dc.Query(fmt.Sprintf("select column_name from information_schema.columns where table_name = '%s' and column_default like 'nextval%%'", tableName))
|
||||
_, res, err := pd.dc.Query(fmt.Sprintf("select column_name from information_schema.columns where table_name = '%s' and column_default like 'nextval%%'", tableName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -307,7 +405,7 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
if colName != "" {
|
||||
|
||||
// 查询自增列当前最大值
|
||||
_, maxRes, err := md.dc.Query(fmt.Sprintf("select max(%s) max_val from %s", colName, tableName))
|
||||
_, maxRes, err := pd.dc.Query(fmt.Sprintf("select max(%s) max_val from %s", colName, tableName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -323,12 +421,12 @@ func (md *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
|
||||
newSeqName := fmt.Sprintf("%s_%s_copy_seq", newTableName, colName)
|
||||
|
||||
// 创建自增序列,当前最大值为旧表最大值
|
||||
_, err = md.dc.Exec(fmt.Sprintf("CREATE SEQUENCE %s START %d INCREMENT 1", newSeqName, maxVal))
|
||||
_, err = pd.dc.Exec(fmt.Sprintf("CREATE SEQUENCE %s START %d INCREMENT 1", newSeqName, maxVal))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 将新表的自增主键序列与主键列相关联
|
||||
_, err = md.dc.Exec(fmt.Sprintf("alter table %s alter column %s set default nextval('%s')", newTableName, colName, newSeqName))
|
||||
_, err = pd.dc.Exec(fmt.Sprintf("alter table %s alter column %s set default nextval('%s')", newTableName, colName, newSeqName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -20,8 +20,9 @@ func init() {
|
||||
dbi.Register(dbi.DbTypeKingbaseEs, meta)
|
||||
dbi.Register(dbi.DbTypeVastbase, meta)
|
||||
|
||||
gauss := new(PostgresMeta)
|
||||
gauss.Param = "dbtype=gauss"
|
||||
gauss := &PostgresMeta{
|
||||
Param: "dbtype=gauss",
|
||||
}
|
||||
dbi.Register(dbi.DbTypeGauss, gauss)
|
||||
}
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ func (sd *SqliteDialect) GetDbProgram() (dbi.DbProgram, error) {
|
||||
return nil, fmt.Errorf("该数据库类型不支持数据库备份与恢复: %v", sd.dc.Info.Type)
|
||||
}
|
||||
|
||||
func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error) {
|
||||
func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any, duplicateStrategy int) (int64, error) {
|
||||
// 执行批量insert sql,跟mysql一样 支持批量insert语法
|
||||
// 生成占位符字符串:如:(?,?)
|
||||
// 重复字符串并用逗号连接
|
||||
@@ -197,7 +197,14 @@ func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
|
||||
// 去除最后一个逗号
|
||||
placeholder = strings.TrimSuffix(repeated, ",")
|
||||
|
||||
sqlStr := fmt.Sprintf("insert into %s (%s) values %s", sd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
prefix := "insert into"
|
||||
if duplicateStrategy == 1 {
|
||||
prefix = "insert or ignore into"
|
||||
} else if duplicateStrategy == 2 {
|
||||
prefix = "insert or replace into"
|
||||
}
|
||||
|
||||
sqlStr := fmt.Sprintf("%s %s (%s) values %s", prefix, sd.dc.Info.Type.QuoteIdentifier(tableName), strings.Join(columns, ","), placeholder)
|
||||
|
||||
// 把二维数组转为一维数组
|
||||
var args []any
|
||||
@@ -210,7 +217,7 @@ func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
|
||||
}
|
||||
|
||||
func (sd *SqliteDialect) GetDataConverter() dbi.DataConverter {
|
||||
return new(DataConverter)
|
||||
return converter
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -218,6 +225,8 @@ var (
|
||||
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit|real`)
|
||||
// 日期时间类型
|
||||
datetimeRegexp = regexp.MustCompile(`(?i)datetime`)
|
||||
|
||||
converter = new(DataConverter)
|
||||
)
|
||||
|
||||
type DataConverter struct {
|
||||
|
||||
@@ -31,6 +31,7 @@ type DataSyncTask struct {
|
||||
TargetTagPath string `orm:"column(target_tag_path)" json:"targetTagPath"`
|
||||
TargetTableName string `orm:"column(target_table_name)" json:"targetTableName"`
|
||||
FieldMap string `orm:"column(field_map)" json:"fieldMap"` // 字段映射json
|
||||
DuplicateStrategy int `orm:"column(duplicate_strategy)" json:"duplicateStrategy"` // 冲突策略 -1:无,1:忽略,2:覆盖
|
||||
}
|
||||
|
||||
func (d *DataSyncTask) TableName() string {
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"mayfly-go/pkg/scheduler"
|
||||
"mayfly-go/pkg/utils/netx"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
@@ -86,14 +85,12 @@ func (stm *SshTunnelMachine) OpenSshTunnel(id string, ip string, port int) (expo
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
// debug
|
||||
//hostname = "0.0.0.0"
|
||||
|
||||
localAddr := fmt.Sprintf("%s:%d", hostname, localPort)
|
||||
localHost := "127.0.0.1"
|
||||
localAddr := fmt.Sprintf("%s:%d", localHost, localPort)
|
||||
listener, err := net.Listen("tcp", localAddr)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -102,7 +99,7 @@ func (stm *SshTunnelMachine) OpenSshTunnel(id string, ip string, port int) (expo
|
||||
tunnel = &Tunnel{
|
||||
id: id,
|
||||
machineId: stm.machineId,
|
||||
localHost: hostname,
|
||||
localHost: localHost,
|
||||
localPort: localPort,
|
||||
remoteHost: ip,
|
||||
remotePort: port,
|
||||
|
||||
Binary file not shown.
@@ -39,6 +39,7 @@ CREATE TABLE IF NOT EXISTS "t_db" (
|
||||
"modifier" text(32),
|
||||
"is_deleted" integer(8) NOT NULL,
|
||||
"delete_time" datetime,
|
||||
"flow_procdef_key" text(64),
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
@@ -164,6 +165,7 @@ CREATE TABLE IF NOT EXISTS "t_db_data_sync_task" (
|
||||
"recent_state" integer(1) NOT NULL,
|
||||
"task_key" text(100),
|
||||
"running_state" integer(1),
|
||||
"duplicate_strategy" integer(1),
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
@@ -267,6 +269,9 @@ CREATE TABLE IF NOT EXISTS "t_db_sql_exec" (
|
||||
"modifier_id" integer(20) NOT NULL,
|
||||
"is_deleted" integer(8) NOT NULL,
|
||||
"delete_time" datetime,
|
||||
"status" integer(8),
|
||||
"flow_biz_key" text(128),
|
||||
"res" text(1000),
|
||||
PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
@@ -842,6 +847,12 @@ INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight,
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709045735, 1708910975, '6egfEVYr/3r3hHEub/', 1, 1, '我的任务', 'procinst-tasks', 1708911263, '{"component":"flow/ProcinstTaskList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstTaskList"}', 1, 'admin', 1, 'admin', '2024-02-27 22:55:35', '2024-02-27 22:56:35', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708911264, 1708910975, '6egfEVYr/fw0Hhvye/', 1, 1, '流程定义', 'procdefs', 1708911264, '{"component":"flow/ProcdefList","icon":"List","isKeepAlive":true,"routeName":"ProcdefList"}', 1, 'admin', 1, 'admin', '2024-02-26 09:34:24', '2024-02-27 22:54:32', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708910975, 0, '6egfEVYr/', 1, 1, '工单流程', '/flow', 60000000, '{"icon":"List","isKeepAlive":true,"routeName":"flow"}', 1, 'admin', 1, 'admin', '2024-02-26 09:29:36', '2024-02-26 15:37:52', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709208354, 1708911264, '6egfEVYr/fw0Hhvye/b4cNf3iq/', 2, 1, '删除流程', 'flow:procdef:del', 1709208354, 'null', 1, 'admin', 1, 'admin', '2024-02-29 20:05:54', '2024-02-29 20:05:54', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709208339, 1708911264, '6egfEVYr/fw0Hhvye/r9ZMTHqC/', 2, 1, '保存流程', 'flow:procdef:save', 1709208339, 'null', 1, 'admin', 1, 'admin', '2024-02-29 20:05:40', '2024-02-29 20:05:40', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709103180, 1708910975, '6egfEVYr/oNCIbynR/', 1, 1, '我的流程', 'procinsts', 1708911263, '{"component":"flow/ProcinstList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstList"}', 1, 'admin', 1, 'admin', '2024-02-28 14:53:00', '2024-02-29 20:36:07', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1709045735, 1708910975, '6egfEVYr/3r3hHEub/', 1, 1, '我的任务', 'procinst-tasks', 1708911263, '{"component":"flow/ProcinstTaskList","icon":"Tickets","isKeepAlive":true,"routeName":"ProcinstTaskList"}', 1, 'admin', 1, 'admin', '2024-02-27 22:55:35', '2024-02-27 22:56:35', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708911264, 1708910975, '6egfEVYr/fw0Hhvye/', 1, 1, '流程定义', 'procdefs', 1708911264, '{"component":"flow/ProcdefList","icon":"List","isKeepAlive":true,"routeName":"ProcdefList"}', 1, 'admin', 1, 'admin', '2024-02-26 09:34:24', '2024-02-27 22:54:32', 0, NULL);
|
||||
INSERT INTO t_sys_resource (id, pid, ui_path, type, status, name, code, weight, meta, creator_id, creator, modifier_id, modifier, create_time, update_time, is_deleted, delete_time) VALUES(1708910975, 0, '6egfEVYr/', 1, 1, '工单流程', '/flow', 60000000, '{"icon":"List","isKeepAlive":true,"routeName":"flow"}', 1, 'admin', 1, 'admin', '2024-02-26 09:29:36', '2024-02-26 15:37:52', 0, NULL);
|
||||
|
||||
|
||||
-- Table: t_sys_role
|
||||
@@ -988,6 +999,66 @@ CREATE TABLE IF NOT EXISTS "t_team_member" (
|
||||
);
|
||||
INSERT INTO t_team_member (id, team_id, account_id, username, create_time, creator_id, creator, update_time, modifier_id, modifier, is_deleted, delete_time) VALUES (7, 3, 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', 0, NULL);
|
||||
|
||||
CREATE TABLE "t_flow_procdef" (
|
||||
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
"def_key" text(100) NOT NULL ,
|
||||
"name" text(200) DEFAULT NULL ,
|
||||
"status" tinyint DEFAULT NULL ,
|
||||
"tasks" text(3000) NOT NULL ,
|
||||
"remark" text(191) DEFAULT NULL,
|
||||
"create_time" datetime NOT NULL,
|
||||
"creator" text(191) DEFAULT NULL,
|
||||
"creator_id" integer(20) NOT NULL,
|
||||
"update_time" datetime NOT NULL,
|
||||
"modifier" text(191) DEFAULT NULL,
|
||||
"modifier_id" integer(20) NOT NULL,
|
||||
"is_deleted" tinyint DEFAULT '0',
|
||||
"delete_time" datetime DEFAULT NULL
|
||||
) ;
|
||||
|
||||
CREATE TABLE "t_flow_procinst" (
|
||||
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
"procdef_id" integer(20) NOT NULL ,
|
||||
"procdef_name" text(64) NOT NULL ,
|
||||
"task_key" text(100) DEFAULT NULL ,
|
||||
"status" tinyint DEFAULT NULL ,
|
||||
"biz_type" text(64) NOT NULL ,
|
||||
"biz_key" text(64) NOT NULL ,
|
||||
"biz_status" tinyint DEFAULT NULL ,
|
||||
"biz_handle_res" text(100) DEFAULT NULL ,
|
||||
"remark" text(191) DEFAULT NULL,
|
||||
"end_time" datetime DEFAULT NULL ,
|
||||
"duration" integer(20) DEFAULT NULL ,
|
||||
"create_time" datetime NOT NULL ,
|
||||
"creator" text(191) DEFAULT NULL ,
|
||||
"creator_id" integer(20) NOT NULL,
|
||||
"update_time" datetime NOT NULL,
|
||||
"modifier" text(191) DEFAULT NULL,
|
||||
"modifier_id" integer(20) NOT NULL,
|
||||
"is_deleted" tinyint DEFAULT '0',
|
||||
"delete_time" datetime DEFAULT NULL
|
||||
) ;
|
||||
|
||||
CREATE TABLE "t_flow_procinst_task" (
|
||||
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
"procinst_id" integer(20) NOT NULL ,
|
||||
"task_key" text(64) NOT NULL ,
|
||||
"task_name" text(64) DEFAULT NULL ,
|
||||
"assignee" text(64) DEFAULT NULL ,
|
||||
"status" tinyint DEFAULT NULL ,
|
||||
"remark" text(191) DEFAULT NULL,
|
||||
"end_time" datetime DEFAULT NULL ,
|
||||
"duration" integer(20) DEFAULT NULL ,
|
||||
"create_time" datetime NOT NULL ,
|
||||
"creator" text(191) DEFAULT NULL,
|
||||
"creator_id" integer(20) NOT NULL,
|
||||
"update_time" datetime NOT NULL,
|
||||
"modifier" text(191) DEFAULT NULL,
|
||||
"modifier_id" integer(20) NOT NULL,
|
||||
"is_deleted" tinyint DEFAULT '0',
|
||||
"delete_time" datetime DEFAULT NULL
|
||||
)
|
||||
|
||||
-- Index: idx_db_backup_id
|
||||
CREATE INDEX IF NOT EXISTS "idx_db_backup_id"
|
||||
ON "t_db_backup_history" (
|
||||
@@ -1006,5 +1077,17 @@ ON "t_db_backup" (
|
||||
"db_name" ASC
|
||||
);
|
||||
|
||||
-- Index: idx_procdef_id
|
||||
CREATE INDEX IF NOT EXISTS "idx_procdef_id"
|
||||
ON "t_flow_procinst" (
|
||||
"procdef_id" ASC
|
||||
);
|
||||
-- Index: idx_procinst_id
|
||||
CREATE INDEX IF NOT EXISTS "idx_procinst_id"
|
||||
ON "t_flow_procinst_task" (
|
||||
"procinst_id" ASC
|
||||
);
|
||||
|
||||
|
||||
COMMIT TRANSACTION;
|
||||
PRAGMA foreign_keys = on;
|
||||
|
||||
@@ -282,6 +282,7 @@ CREATE TABLE `t_db_data_sync_task`
|
||||
`recent_state` tinyint(1) NOT NULL DEFAULT '0' COMMENT '最近一次状态 0未执行 1成功 2失败',
|
||||
`task_key` varchar(100) DEFAULT NULL COMMENT '定时任务唯一uuid key',
|
||||
`running_state` tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止',
|
||||
`duplicate_strategy`tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖',
|
||||
PRIMARY KEY (`id`)
|
||||
) COMMENT='数据同步';
|
||||
|
||||
|
||||
@@ -11,68 +11,68 @@ INSERT INTO t_sys_resource (id, pid, ui_path, `type`, status, name, code, weight
|
||||
-- 工单流程相关表
|
||||
CREATE TABLE `t_flow_procdef` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`def_key` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '流程定义key',
|
||||
`name` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '流程名称',
|
||||
`def_key` varchar(100) NOT NULL COMMENT '流程定义key',
|
||||
`name` varchar(191) DEFAULT NULL COMMENT '流程名称',
|
||||
`status` tinyint DEFAULT NULL COMMENT '状态',
|
||||
`tasks` varchar(3000) COLLATE utf8mb4_bin NOT NULL COMMENT '审批节点任务信息',
|
||||
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`remark` varchar(191) DEFAULT NULL,
|
||||
`create_time` datetime NOT NULL,
|
||||
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`creator` varchar(191) DEFAULT NULL,
|
||||
`creator_id` bigint NOT NULL,
|
||||
`update_time` datetime NOT NULL,
|
||||
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`modifier` varchar(191) DEFAULT NULL,
|
||||
`modifier_id` bigint NOT NULL,
|
||||
`is_deleted` tinyint DEFAULT '0',
|
||||
`delete_time` datetime DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程定义';
|
||||
) COMMENT='流程-流程定义';
|
||||
|
||||
CREATE TABLE `t_flow_procinst` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`procdef_id` bigint NOT NULL COMMENT '流程定义id',
|
||||
`procdef_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '流程定义名称',
|
||||
`task_key` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '当前任务key',
|
||||
`procdef_name` varchar(64) NOT NULL COMMENT '流程定义名称',
|
||||
`task_key` varchar(100) DEFAULT NULL COMMENT '当前任务key',
|
||||
`status` tinyint DEFAULT NULL COMMENT '状态',
|
||||
`biz_type` varchar(64) COLLATE utf8mb4_bin NOT NULL COMMENT '关联业务类型',
|
||||
`biz_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '关联业务key',
|
||||
`biz_key` varchar(64) NOT NULL COMMENT '关联业务key',
|
||||
`biz_status` tinyint DEFAULT NULL COMMENT '业务状态',
|
||||
`biz_handle_res` varchar(100) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '关联的业务处理结果',
|
||||
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`remark` varchar(191) DEFAULT NULL,
|
||||
`end_time` datetime DEFAULT NULL COMMENT '结束时间',
|
||||
`duration` bigint DEFAULT NULL COMMENT '流程持续时间(开始到结束)',
|
||||
`create_time` datetime NOT NULL COMMENT '流程发起时间',
|
||||
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '流程发起人',
|
||||
`creator` varchar(191) DEFAULT NULL COMMENT '流程发起人',
|
||||
`creator_id` bigint NOT NULL,
|
||||
`update_time` datetime NOT NULL,
|
||||
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`modifier` varchar(191) DEFAULT NULL,
|
||||
`modifier_id` bigint NOT NULL,
|
||||
`is_deleted` tinyint DEFAULT '0',
|
||||
`delete_time` datetime DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_procdef_id` (`procdef_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程实例(根据流程定义开启一个流程)';
|
||||
) COMMENT='流程-流程实例(根据流程定义开启一个流程)';
|
||||
|
||||
CREATE TABLE `t_flow_procinst_task` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`procinst_id` bigint NOT NULL COMMENT '流程实例id',
|
||||
`task_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '任务key',
|
||||
`task_name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '任务名称',
|
||||
`assignee` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT '分配到该任务的用户',
|
||||
`task_key` varchar(64) NOT NULL COMMENT '任务key',
|
||||
`task_name` varchar(64) DEFAULT NULL COMMENT '任务名称',
|
||||
`assignee` varchar(64) DEFAULT NULL COMMENT '分配到该任务的用户',
|
||||
`status` tinyint DEFAULT NULL COMMENT '状态',
|
||||
`remark` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`remark` varchar(191) DEFAULT NULL,
|
||||
`end_time` datetime DEFAULT NULL COMMENT '结束时间',
|
||||
`duration` bigint DEFAULT NULL COMMENT '任务持续时间(开始到结束)',
|
||||
`create_time` datetime NOT NULL COMMENT '任务开始时间',
|
||||
`creator` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`creator` varchar(191) DEFAULT NULL,
|
||||
`creator_id` bigint NOT NULL,
|
||||
`update_time` datetime NOT NULL,
|
||||
`modifier` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`modifier` varchar(191) DEFAULT NULL,
|
||||
`modifier_id` bigint NOT NULL,
|
||||
`is_deleted` tinyint DEFAULT '0',
|
||||
`delete_time` datetime DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_procinst_id` (`procinst_id`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='流程-流程实例任务';
|
||||
) COMMENT='流程-流程实例任务';
|
||||
|
||||
-- 新增工单流程相关字段
|
||||
ALTER TABLE t_db_sql_exec ADD status tinyint NULL COMMENT '执行状态';
|
||||
@@ -82,6 +82,9 @@ ALTER TABLE t_db_sql_exec ADD res varchar(1000) NULL COMMENT '执行结果';
|
||||
ALTER TABLE t_db ADD flow_procdef_key varchar(64) NULL COMMENT '审批流-流程定义key(有值则说明关键操作需要进行审批执行)';
|
||||
|
||||
-- 历史执行记录调整为成功状态
|
||||
UPDATE t_db_sql_exec SET status = 2
|
||||
UPDATE t_db_sql_exec SET status = 2;
|
||||
|
||||
ALTER TABLE `t_db_data_sync_task`
|
||||
ADD COLUMN `duplicate_strategy` tinyint(1) NULL DEFAULT -1 comment '冲突策略 -1:无,1:忽略,2:覆盖';
|
||||
|
||||
commit;
|
||||
Reference in New Issue
Block a user