!123 一些bug修复

* fix: 数据同步、数据迁移体验优化
* fix: des加密传输sql
* fix: 修复达梦字段注释显示问题
* fix: mysql timestamp 字段类型导出ddl错误修复
This commit is contained in:
zongyangleo
2024-08-22 00:43:39 +00:00
committed by Coder慌
parent 2deb3109c2
commit 43edef412c
26 changed files with 226 additions and 50 deletions

View File

@@ -5,4 +5,6 @@ VITE_PORT = 8889
VITE_OPEN = false
# public path 配置线上环境路径(打包)
VITE_PUBLIC_PATH = ''
VITE_PUBLIC_PATH = ''
VITE_EDITOR=idea

View File

@@ -42,6 +42,7 @@
"xterm-addon-web-links": "^0.9.0"
},
"devDependencies": {
"@types/crypto-js": "^4.2.2",
"@types/lodash": "^4.14.178",
"@types/node": "^18.14.0",
"@types/nprogress": "^0.2.0",
@@ -51,6 +52,7 @@
"@vitejs/plugin-vue": "^5.0.5",
"@vue/compiler-sfc": "^3.4.32",
"code-inspector-plugin": "^0.4.5",
"crypto-js": "^4.2.0",
"dotenv": "^16.3.1",
"eslint": "^8.35.0",
"eslint-plugin-vue": "^9.25.0",

View File

@@ -69,7 +69,7 @@ class Api {
*/
async xhrReq(param: any = null, options: any = {}): Promise<any> {
if (this.beforeHandler) {
this.beforeHandler(param);
await this.beforeHandler(param);
}
return request.xhrReq(this.method, this.url, param, options);
}

View File

@@ -0,0 +1,48 @@
import CryptoJS from 'crypto-js';
import { getToken } from '@/common/utils/storage';
/**
* AES 加密数据
* @param word
* @param key
*/
export function DesEncrypt(word: string, key?: string) {
if (!key) {
key = getToken().substring(0, 24);
}
const srcs = CryptoJS.enc.Utf8.parse(word);
const iv = CryptoJS.lib.WordArray.random(8); // 生成随机IV
const encrypted = CryptoJS.TripleDES.encrypt(srcs, CryptoJS.enc.Utf8.parse(key), {
iv: iv,
mode: CryptoJS.mode.CBC,
padding: CryptoJS.pad.Pkcs7,
});
return iv.concat(encrypted.ciphertext).toString(CryptoJS.enc.Base64);
}
/**
* AES 解密 :字符串 key iv 返回base64
* */
export function DesDecrypt(encryptedData: string, key?: string) {
if (!key) {
key = getToken().substring(0, 32);
}
// 解码Base64
const ciphertext = CryptoJS.enc.Base64.parse(encryptedData);
// 分离IV和加密数据
const iv = ciphertext.clone();
iv.sigBytes = 8;
iv.clamp();
ciphertext.words.splice(0, 2); // 移除IV
ciphertext.sigBytes -= 8;
const decrypted = CryptoJS.TripleDES.decrypt({ ciphertext } as any, CryptoJS.enc.Utf8.parse(key), {
iv: iv,
mode: CryptoJS.mode.CBC,
padding: CryptoJS.pad.Pkcs7,
});
return decrypted.toString(CryptoJS.enc.Utf8);
}

View File

@@ -4,6 +4,7 @@ export interface ViteEnv {
VITE_PORT: number;
VITE_OPEN: boolean;
VITE_PUBLIC_PATH: string;
VITE_EDITOR: string;
}
export function loadEnv(): ViteEnv {

View File

@@ -42,7 +42,7 @@ const useCustomFetch = createFetch({
export function useApiFetch<T>(api: Api, params: any = null, reqOptions: RequestInit = {}) {
const uaf = useCustomFetch<T>(api.url, {
beforeFetch({ url, options }) {
async beforeFetch({ url, options }) {
options.method = api.method;
if (!params) {
return;
@@ -57,7 +57,7 @@ export function useApiFetch<T>(api: Api, params: any = null, reqOptions: Request
}
if (api.beforeHandler) {
paramsValue = api.beforeHandler(paramsValue);
paramsValue = await api.beforeHandler(paramsValue);
}
if (paramsValue) {

View File

@@ -12,6 +12,9 @@
<el-form :model="form" ref="dbForm" :rules="rules" label-width="auto">
<el-tabs v-model="tabActiveName">
<el-tab-pane label="基本信息" :name="basicTab">
<el-form-item prop="taskName" label="任务名" required>
<el-input v-model.trim="form.taskName" placeholder="请输入任务名" auto-complete="off" />
</el-form-item>
<el-form-item prop="srcDbId" label="源数据库" required>
<db-select-tree
placeholder="请选择源数据库"
@@ -117,6 +120,7 @@ const tableTab = 'table';
type FormData = {
id?: number;
taskName: string;
srcDbId?: number;
srcDbName?: string;
srcDbType?: string;

View File

@@ -14,11 +14,16 @@
<el-button v-auth="perms.del" :disabled="selectionData.length < 1" @click="del()" type="danger" icon="delete">删除</el-button>
</template>
<template #taskName="{ data }">
<span :style="`${data.taskName ? '' : 'color:red'}`">
{{ data.taskName || '请设置' }}
</span>
</template>
<template #srcDb="{ data }">
<el-tooltip :content="`${data.srcTagPath} > ${data.srcInstName} > ${data.srcDbName}`">
<span>
<SvgIcon :name="getDbDialect(data.srcDbType).getInfo().icon" :size="18" />
{{ data.srcInstName }}
{{ data.srcDbName }}
</span>
</el-tooltip>
</template>
@@ -26,7 +31,7 @@
<el-tooltip :content="`${data.targetTagPath} > ${data.targetInstName} > ${data.targetDbName}`">
<span>
<SvgIcon :name="getDbDialect(data.targetDbType).getInfo().icon" :size="18" />
{{ data.targetInstName }}
{{ data.targetDbName }}
</span>
</el-tooltip>
</template>
@@ -71,8 +76,9 @@ const perms = {
const searchItems = [SearchItem.input('name', '名称')];
const columns = ref([
TableColumn.new('srcDb', '源库').setMinWidth(200).isSlot(),
TableColumn.new('targetDb', '目标库').setMinWidth(200).isSlot(),
TableColumn.new('taskName', '任务名').setMinWidth(150).isSlot(),
TableColumn.new('srcDb', '库').setMinWidth(150).isSlot(),
TableColumn.new('targetDb', '目标库').setMinWidth(150).isSlot(),
TableColumn.new('runningState', '执行状态').typeTag(DbTransferRunningStateEnum),
TableColumn.new('creator', '创建人'),
TableColumn.new('createTime', '创建时间').isTime(),

View File

@@ -328,7 +328,7 @@ watch(dialogVisible, async (newValue: boolean) => {
db.databases = db.database?.split(' ').sort() || [];
state.srcDbInst = DbInst.getOrNewInst(db);
state.form.srcDbType = state.srcDbInst.type;
state.form.srcInstName = db.instanceName;
state.form.srcInstName = db.name;
}
// 初始化target数据源
@@ -340,7 +340,7 @@ watch(dialogVisible, async (newValue: boolean) => {
db.databases = db.database?.split(' ').sort() || [];
state.targetDbInst = DbInst.getOrNewInst(db);
state.form.targetDbType = state.targetDbInst.type;
state.form.targetInstName = db.instanceName;
state.form.targetInstName = db.name;
}
if (targetDbId && state.form.targetDbName) {

View File

@@ -1,5 +1,5 @@
import Api from '@/common/Api';
import { Base64 } from 'js-base64';
import { DesEncrypt } from '@/common/des';
export const dbApi = {
// 获取权限列表
@@ -16,20 +16,7 @@ export const dbApi = {
pgSchemas: Api.newGet('/dbs/{id}/pg/schemas'),
// 获取表即列提示
hintTables: Api.newGet('/dbs/{id}/hint-tables'),
sqlExec: Api.newPost('/dbs/{id}/exec-sql').withBeforeHandler((param: any) => {
// sql编码处理
if (param.sql) {
// 判断是开发环境就打印sql
if (process.env.NODE_ENV === 'development') {
console.log(param.sql);
}
// 非base64编码sql则进行base64编码refreshToken时会重复调用该方法故简单判断下
if (!Base64.isValid(param.sql)) {
param.sql = Base64.encode(param.sql);
}
}
return param;
}),
sqlExec: Api.newPost('/dbs/{id}/exec-sql').withBeforeHandler(async (param: any) => await encryptField(param, 'sql')),
// 保存sql
saveSql: Api.newPost('/dbs/{id}/sql'),
// 获取保存的sql
@@ -73,13 +60,7 @@ export const dbApi = {
// 数据同步相关
datasyncTasks: Api.newGet('/datasync/tasks'),
saveDatasyncTask: Api.newPost('/datasync/tasks/save').withBeforeHandler((param: any) => {
// sql编码处理
if (param.dataSql) {
param.dataSql = Base64.encode(param.dataSql);
}
return param;
}),
saveDatasyncTask: Api.newPost('/datasync/tasks/save').withBeforeHandler(async (param: any) => await encryptField(param, 'dataSql')),
getDatasyncTask: Api.newGet('/datasync/tasks/{taskId}'),
deleteDatasyncTask: Api.newDelete('/datasync/tasks/{taskId}/del'),
updateDatasyncTaskStatus: Api.newPost('/datasync/tasks/{taskId}/status'),
@@ -100,3 +81,17 @@ export const dbSqlExecApi = {
// 根据业务key获取sql执行信息
getSqlExecByBizKey: Api.newGet('/dbs/sql-execs'),
};
const encryptField = async (param: any, field: string) => {
// sql编码处理
if (!param['_encrypted'] && param[field]) {
// 判断是开发环境就打印sql
if (process.env.NODE_ENV === 'development') {
console.log(param[field]);
}
// 使用rsa公钥加密sql
param['_encrypted'] = 1;
param[field] = DesEncrypt(param[field]);
// console.log('解密结果', DesDecrypt(param[field]));
}
return param;
};

View File

@@ -8,7 +8,7 @@ const pathResolve = (dir: string): any => {
return resolve(__dirname, '.', dir);
};
const { VITE_PORT, VITE_OPEN, VITE_PUBLIC_PATH } = loadEnv();
const { VITE_PORT, VITE_OPEN, VITE_PUBLIC_PATH, VITE_EDITOR } = loadEnv();
const alias: Record<string, string> = {
'@': pathResolve('src/'),
@@ -19,6 +19,7 @@ const viteConfig: UserConfig = {
vue(),
CodeInspectorPlugin({
bundler: 'vite',
editor: VITE_EDITOR,
}),
],
root: process.cwd(),

View File

@@ -2,7 +2,6 @@ package api
import (
"context"
"encoding/base64"
"fmt"
"io"
"mayfly-go/internal/db/api/form"
@@ -24,6 +23,7 @@ import (
"mayfly-go/pkg/req"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/cryptox"
"mayfly-go/pkg/utils/stringx"
"mayfly-go/pkg/ws"
"strings"
@@ -103,17 +103,18 @@ func (d *Db) DeleteDb(rc *req.Ctx) {
func (d *Db) ExecSql(rc *req.Ctx) {
form := req.BindJsonAndValid(rc, new(form.DbSqlExecForm))
ui := rc.GetLoginAccount()
dbId := getDbId(rc)
dbConn, err := d.DbApp.GetDbConn(dbId, form.Db)
biz.ErrIsNil(err)
biz.ErrIsNilAppendErr(d.TagApp.CanAccess(rc.GetLoginAccount().Id, dbConn.Info.CodePath...), "%s")
global.EventBus.Publish(rc.MetaCtx, event.EventTopicResourceOp, dbConn.Info.CodePath[0])
sqlBytes, err := base64.StdEncoding.DecodeString(form.Sql)
sqlStr, err := cryptox.DesDecryptByToken(form.Sql, ui.Token)
biz.ErrIsNilAppendErr(err, "sql解码失败: %s")
// 去除前后空格及换行符
sql := stringx.TrimSpaceAndBr(string(sqlBytes))
sql := stringx.TrimSpaceAndBr(sqlStr)
rc.ReqParam = fmt.Sprintf("%s %s\n-> %s", dbConn.Info.GetLogDesc(), form.ExecId, sql)
biz.NotEmpty(form.Sql, "sql不能为空")

View File

@@ -1,13 +1,13 @@
package api
import (
"encoding/base64"
"mayfly-go/internal/db/api/form"
"mayfly-go/internal/db/api/vo"
"mayfly-go/internal/db/application"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/req"
"mayfly-go/pkg/utils/cryptox"
"mayfly-go/pkg/utils/stringx"
"strconv"
"strings"
@@ -36,9 +36,9 @@ func (d *DataSyncTask) SaveTask(rc *req.Ctx) {
task := req.BindJsonAndCopyTo[*entity.DataSyncTask](rc, form, new(entity.DataSyncTask))
// 解码base64 sql
sqlBytes, err := base64.StdEncoding.DecodeString(task.DataSql)
sqlStr, err := cryptox.DesDecryptByToken(task.DataSql, rc.GetLoginAccount().Token)
biz.ErrIsNilAppendErr(err, "sql解码失败: %s")
sql := stringx.TrimSpaceAndBr(string(sqlBytes))
sql := stringx.TrimSpaceAndBr(sqlStr)
task.DataSql = sql
form.DataSql = sql

View File

@@ -2,6 +2,7 @@ package form
type DbTransferTaskForm struct {
Id uint64 `json:"id"`
TaskName string `binding:"required" json:"taskName"` // 任务名称
CheckedKeys string `binding:"required" json:"checkedKeys"` // 选中需要迁移的表
DeleteTable int `binding:"required" json:"deleteTable"` // 创建表前是否删除表 1是 2否
NameCase int `binding:"required" json:"nameCase"` // 表名、字段大小写转换 1无 2大写 3小写

View File

@@ -11,6 +11,7 @@ type DbTransferTaskListVO struct {
RunningState int `json:"runningState"`
LogId uint64 `json:"logId"`
TaskName string `json:"taskName"` // 任务名称
CheckedKeys string `json:"checkedKeys"` // 选中需要迁移的表
DeleteTable int `json:"deleteTable"` // 创建表前是否删除表

View File

@@ -60,15 +60,15 @@ select a.owner,
a.char_col_decl_length as CHAR_MAX_LENGTH,
a.data_precision as NUM_PRECISION,
a.data_scale as NUM_SCALE,
b.comments as COLUMN_COMMENT,
b.COMMENT$ as COLUMN_COMMENT,
a.data_default as COLUMN_DEFAULT,
case when t.INFO2 & 0x01 = 0x01 then 1 else 0 end as IS_IDENTITY,
case when t2.constraint_type = 'P' then 1 else 0 end as IS_PRIMARY_KEY
from all_tab_columns a
left join all_col_comments b
on b.owner = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
and b.table_name = a.table_name
and a.column_name = b.column_name
left join SYSCOLUMNCOMMENTS b
on b.SCHNAME = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
and b.TVNAME = a.table_name
and a.column_name = b.COLNAME
left join (select c1.*, c2.object_name, c2.owner
FROM SYS.SYSCOLUMNS c1
join SYS.all_objects c2 on c1.id = c2.object_id and c2.object_type = 'TABLE') t

View File

@@ -217,9 +217,17 @@ func (md *MysqlMetaData) genColumnBasicSql(column dbi.Column) string {
if !column.Nullable {
nullAble = " NOT NULL"
}
columnType := column.GetColumnType()
if nullAble == "" && strings.Contains(columnType, "timestamp") {
nullAble = " NULL"
}
defVal := "" // 默认值需要判断引号,如函数是不需要引号的 // 为了防止跨源函数不支持 当默认值是函数时,不需要设置默认值
if column.ColumnDefault != "" && !strings.Contains(column.ColumnDefault, "(") {
defVal := "" // 默认值需要判断引号,如函数是不需要引号的
if column.ColumnDefault != "" &&
// 当默认值是字符串'NULL'时,不需要设置默认值
column.ColumnDefault != "NULL" &&
// 为了防止跨源函数不支持 当默认值是函数时,不需要设置默认值
!strings.Contains(column.ColumnDefault, "(") {
// 哪些字段类型默认值需要加引号
mark := false
if collx.ArrayAnyMatches([]string{"char", "text", "date", "time", "lob"}, strings.ToLower(dataType)) {
@@ -244,7 +252,7 @@ func (md *MysqlMetaData) genColumnBasicSql(column dbi.Column) string {
comment = fmt.Sprintf(" COMMENT '%s'", commentStr)
}
columnSql := fmt.Sprintf(" %s %s%s%s%s%s", md.dc.GetMetaData().QuoteIdentifier(column.ColumnName), column.GetColumnType(), nullAble, incr, defVal, comment)
columnSql := fmt.Sprintf(" %s %s%s%s%s%s", md.dc.GetMetaData().QuoteIdentifier(column.ColumnName), columnType, nullAble, incr, defVal, comment)
return columnSql
}

View File

@@ -9,6 +9,7 @@ type DbTransferTask struct {
RunningState DbTransferRunningState `orm:"column(running_state)" json:"runningState"` // 运行状态
LogId uint64 `json:"logId"`
TaskName string `orm:"column(task_name)" json:"taskName"` // 任务名称
CheckedKeys string `orm:"column(checked_keys)" json:"checkedKeys"` // 选中需要迁移的表
DeleteTable int `orm:"column(delete_table)" json:"deleteTable"` // 创建表前是否删除表

View File

@@ -19,6 +19,7 @@ type DataSyncLogQuery struct {
}
type DbTransferTaskQuery struct {
Name string `json:"name" form:"name"`
}
type DbTransferLogQuery struct {

View File

@@ -17,8 +17,8 @@ func newDbTransferTaskRepo() repository.DbTransferTask {
// 分页获取数据库信息列表
func (d *dbTransferTaskRepoImpl) GetTaskList(condition *entity.DbTransferTaskQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
qd := model.NewCond()
//Like("task_name", condition.Name).
qd := model.NewCond().
Like("task_name", condition.Name)
//Eq("status", condition.Status)
return d.PageByCondToAny(qd, pageParam, toEntity)
}

View File

@@ -3,4 +3,5 @@ package model
type LoginAccount struct {
Id uint64
Username string
Token string
}

View File

@@ -67,6 +67,7 @@ func PermissionHandler(rc *Ctx) error {
rc.MetaCtx = contextx.WithLoginAccount(rc.MetaCtx, &model.LoginAccount{
Id: userId,
Username: userName,
Token: tokenStr,
})
return nil
}

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/des"
"crypto/md5"
"crypto/rand"
"crypto/rsa"
@@ -12,6 +13,7 @@ import (
"encoding/hex"
"encoding/pem"
"errors"
"fmt"
"mayfly-go/pkg/cache"
"mayfly-go/pkg/logx"
"os"
@@ -279,6 +281,68 @@ func AesDecryptBase64(data string, key []byte) ([]byte, error) {
return AesDecrypt(dataByte, key)
}
// DES加密函数
func DesEncrypt(data, key []byte) (string, error) {
block, err := des.NewTripleDESCipher(key)
if err != nil {
return "", err
}
// 对数据进行填充
data = pkcs7Padding(data, des.BlockSize)
// 创建一个初始化向量
iv := make([]byte, des.BlockSize)
if _, err := rand.Read(iv); err != nil {
return "", err
}
// 创建加密器
mode := cipher.NewCBCEncrypter(block, iv)
// 加密
encrypted := make([]byte, len(data))
mode.CryptBlocks(encrypted, data)
// 将IV和加密数据组合
result := append(iv, encrypted...)
// 使用Base64编码
return base64.StdEncoding.EncodeToString(result), nil
}
func DesDecryptByToken(data string, token string) (string, error) {
key := []byte(token[:24])
return DesDecrypt(data, key)
}
func DesDecrypt(data string, key []byte) (string, error) {
// Base64解码
ciphertext, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
block, err := des.NewTripleDESCipher(key)
if err != nil {
return "", err
}
// 确保密文长度正确
if len(ciphertext) < des.BlockSize {
return "", fmt.Errorf("ciphertext too short")
}
// 提取IV
iv := ciphertext[:des.BlockSize]
ciphertext = ciphertext[des.BlockSize:]
// 创建解密器
mode := cipher.NewCBCDecrypter(block, iv)
// 解密仍然用已存在的切片接收结果,无需重新创建切片
mode.CryptBlocks(ciphertext, ciphertext)
// 去除填充
res, err := pkcs7UnPadding(ciphertext)
return string(res), err
}
// pkcs7Padding 填充
func pkcs7Padding(data []byte, blockSize int) []byte {
//判断缺少几位长度。最少1最多 blockSize

View File

@@ -0,0 +1,35 @@
package cryptox
import (
"encoding/base64"
"testing"
)
func TestAesEncrypt(t *testing.T) {
key := []byte("eyJhbGciOiJIUzI1NiIsInR5")
data := []byte("SELECT * FROM \"instruct\" OFFSET 0 LIMIT 25;")
encrypt, err := AesEncrypt(data, key)
if err != nil {
t.Error(err)
}
toString := base64.StdEncoding.EncodeToString(encrypt)
t.Log(toString)
decrypt, err := AesDecrypt(encrypt, key)
t.Log(string(decrypt))
}
func TestDes(t *testing.T) {
key := []byte("eyJhbGciOiJIUzI1NiIsInR5")
data := []byte("SELECT * FROM \"instruct\" OFFSET 0 LIMIT 25;")
encrypt, err := DesEncrypt(data, key)
if err != nil {
t.Error(err)
}
t.Log("encrypt", encrypt)
decrypt, err := DesDecrypt(encrypt, key)
if err != nil {
t.Error(err)
}
t.Log("decrypt", decrypt)
}

View File

@@ -63,6 +63,7 @@ CREATE TABLE `t_db_transfer_task` (
`modifier` varchar(100) NOT NULL COMMENT '修改人姓名',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
`is_deleted` tinyint(1) DEFAULT '0' COMMENT '是否删除',
`task_name` varchar(100) NULL comment '任务名',
`delete_time` datetime DEFAULT NULL COMMENT '删除时间',
`checked_keys` text NOT NULL COMMENT '选中需要迁移的表',
`delete_table` tinyint(4) NOT NULL COMMENT '创建表前是否删除表 1是 -1否',

View File

@@ -0,0 +1,2 @@
ALTER TABLE `t_db_transfer_task`
ADD COLUMN `task_name` varchar(100) NULL comment '任务名' after `delete_time`;