TRKJ/mayfly-go
2
0
Fork 0
mirror of https://gitee.com/dromara/mayfly-go synced 2025-11-02 23:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: dbm重构等

Browse Source
This commit is contained in:
meilin.huang
2024-03-26 21:46:03 +08:00
parent 2acc295259
commit fc166650b3
34 changed files with 1660 additions and 1477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
mayfly_go_web/package.json
View File

@@ -17,7 +17,7 @@
"countup.js": "^2.8.0",
"cropperjs": "^1.6.1",
"echarts": "^5.5.0",
"element-plus": "^2.6.1",
"element-plus": "^2.6.2",
"js-base64": "^3.7.7",
"jsencrypt": "^3.3.2",
"lodash": "^4.17.21",
@@ -56,7 +56,7 @@
"prettier": "^3.2.5",
"sass": "^1.69.0",
"typescript": "^5.3.2",
"vite": "^5.2.2",
"vite": "^5.2.6",
"vue-eslint-parser": "^9.4.2"
},
"browserslist": [

4
mayfly_go_web/src/views/ops/component/TagTree.vue
View File

@@ -10,7 +10,7 @@
:props="treeProps"
lazy
node-key="key"
:expand-on-click-node="true"
:expand-on-click-node="false"
:filter-node-method="filterNode"
@node-click="treeNodeClick"
@node-expand="treeNodeClick"
@@ -140,8 +140,8 @@ const loadNode = async (node: any, resolve: any) => {
};
const treeNodeClick = (data: any) => {
emit('nodeClick', data);
if (!data.disabled && !data.type.nodeDblclickFunc && data.type.nodeClickFunc) {
emit('nodeClick', data);
data.type.nodeClickFunc(data);
}
// 关闭可能存在的右击菜单

16
mayfly_go_web/src/views/ops/db/DbList.vue
View File

@@ -89,8 +89,8 @@
<el-col :span="9">
<el-form-item label="导出内容: ">
<el-checkbox-group v-model="exportDialog.contents" :min="1">
<el-checkbox label="结构" />
<el-checkbox label="数据" />
<el-checkbox label="结构" value="结构" />
<el-checkbox label="数据" value="数据" />
</el-checkbox-group>
</el-form-item>
</el-col>
@@ -220,6 +220,7 @@ import DbBackupList from './DbBackupList.vue';
import DbBackupHistoryList from './DbBackupHistoryList.vue';
import DbRestoreList from './DbRestoreList.vue';
import ResourceTags from '../component/ResourceTags.vue';
import { sleep } from '@/common/utils/loading';
const DbEdit = defineAsyncComponent(() => import('./DbEdit.vue'));
@@ -499,9 +500,8 @@ const onDumpDbs = async (row: any) => {
/**
* 数据库信息导出
*/
const dumpDbs = () => {
const dumpDbs = async () => {
isTrue(state.exportDialog.value.length > 0, '请添加要导出的数据库');
const a = document.createElement('a');
let type = 0;
for (let c of state.exportDialog.contents) {
if (c == '结构') {
@@ -510,13 +510,15 @@ const dumpDbs = () => {
type += 2;
}
}
for (let db of state.exportDialog.value) {
const a = document.createElement('a');
a.setAttribute(
'href',
`${config.baseApiUrl}/dbs/${state.exportDialog.dbId}/dump?db=${state.exportDialog.value.join(',')}&type=${type}&extName=${
state.exportDialog.extName
}&${joinClientParams()}`
`${config.baseApiUrl}/dbs/${state.exportDialog.dbId}/dump?db=${db}&type=${type}&extName=${state.exportDialog.extName}&${joinClientParams()}`
);
a.click();
await sleep(500);
}
state.exportDialog.visible = false;
};

5
mayfly_go_web/src/views/ops/db/SqlExec.vue
View File

@@ -366,7 +366,10 @@ const NodeTypeTableMenu = new NodeType(SqlExecNodeType.TableMenu)
parentNode.params.dbTableSize = dbTableSize == 0 ? '' : formatByteSize(dbTableSize);
return tablesNode;
})
.withNodeClickFunc(nodeClickChangeDb);
.withNodeDblclickFunc((node: TagTreeNode) => {
const params = node.params;
addTablesOpTab({ id: params.id, db: params.db, type: params.type, nodeKey: node.key });
});
// 数据库sql模板菜单节点
const NodeTypeSqlMenu = new NodeType(SqlExecNodeType.SqlMenu)

4
mayfly_go_web/src/views/ops/tag/TagTreeList.vue
View File

@@ -30,7 +30,7 @@
ref="tagTreeRef"
class="none-select"
node-key="id"
:highlight-current="true"
highlight-current
:props="props"
:data="data"
@node-expand="handleNodeExpand"
@@ -193,7 +193,7 @@ const state = reactive({
},
items: [contextmenuEdit, contextmenuAdd, contextmenuDel],
},
activeTabName: 'tagDetail',
activeTabName: TagDetail,
currentTag: null as any,
resourceCount: {} as any,
});

2
mayfly_go_web/src/views/system/enums.ts
View File

@@ -1,7 +1,7 @@
import { EnumValue } from '@/common/Enum';
export const ResourceTypeEnum = {
Menu: EnumValue.of(1, '菜单'),
Menu: EnumValue.of(1, '菜单').tagTypeSuccess(),
Permission: EnumValue.of(2, '权限'),
};

155
mayfly_go_web/src/views/system/resource/ResourceList.vue
View File

@@ -1,13 +1,16 @@
<template>
<div class="card system-resouce-list">
<div class="card pd10 flex-justify-between">
<div>
<el-input v-model="filterResource" clearable placeholder="输入关键字过滤(右击进行操作)" style="width: 220px; margin-right: 10px" />
<el-button v-auth="perms.addResource" type="primary" icon="plus" @click="addResource(false)">添加</el-button>
</div>
<Splitpanes class="default-theme">
<Pane size="25" min-size="20" max-size="30">
<div class="card pd5 mr5">
<el-input v-model="filterResource" clearable placeholder="输入关键字过滤(右击操作)" style="width: 200px; margin-right: 10px" />
<el-button v-auth="perms.addResource" type="primary" icon="plus" @click="addResource(false)"></el-button>
<div>
<span> <SvgIcon name="info-filled" />红色、橙色字体表示禁用状态 (右击资源进行操作) </span>
<div class="fr">
<el-tooltip placement="top">
<template #content> 红色、橙色字体表示禁用状态 (右击资源进行操作) </template>
<span> <SvgIcon name="question-filled" /> </span>
</el-tooltip>
</div>
</div>
<el-scrollbar class="tree-data">
@@ -18,12 +21,13 @@
node-key="id"
:props="props"
:data="data"
highlight-current
@node-expand="handleNodeExpand"
@node-collapse="handleNodeCollapse"
@node-contextmenu="nodeContextmenu"
@node-click="treeNodeClick"
:default-expanded-keys="defaultExpandedKeys"
:expand-on-click-node="true"
:expand-on-click-node="false"
draggable
:allow-drop="allowDrop"
@node-drop="handleDrop"
@@ -47,6 +51,53 @@
</template>
</el-tree>
</el-scrollbar>
</Pane>
<Pane min-size="40">
<div class="ml10">
<el-tabs v-model="state.activeTabName" v-if="currentResource">
<el-tab-pane label="菜单资源详情" :name="ResourceDetail">
<el-descriptions title="资源信息" :column="2" border>
<el-descriptions-item label="类型">
<enum-tag :enums="ResourceTypeEnum" :value="currentResource?.type" />
</el-descriptions-item>
<el-descriptions-item label="名称">{{ currentResource.name }}</el-descriptions-item>
<el-descriptions-item label="code[菜单path]">{{ currentResource.code }}</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="图标">
<SvgIcon :name="currentResource.meta.icon" />
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="路由名">
{{ currentResource.meta.routeName }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="组件路径">
{{ currentResource.meta.component }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="是否缓存">
{{ currentResource.meta.isKeepAlive ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="是否隐藏">
{{ currentResource.meta.isHide ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="tag不可删除">
{{ currentResource.meta.isAffix ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue" label="外链">
{{ currentResource.meta.linkType ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="currentResource.type == menuTypeValue && currentResource.meta.linkType > 0" label="外链">
{{ currentResource.meta.link }}
</el-descriptions-item>
<el-descriptions-item label="创建者">{{ currentResource.creator }}</el-descriptions-item>
<el-descriptions-item label="创建时间">{{ dateFormat(currentResource.createTime) }} </el-descriptions-item>
<el-descriptions-item label="修改者">{{ currentResource.modifier }}</el-descriptions-item>
<el-descriptions-item label="更新时间">{{ dateFormat(currentResource.updateTime) }} </el-descriptions-item>
</el-descriptions>
</el-tab-pane>
</el-tabs>
</div>
</Pane>
</Splitpanes>
<ResourceEdit
:title="dialogForm.title"
@@ -58,45 +109,6 @@
@val-change="valChange"
/>
<el-dialog v-model="infoDialog.visible">
<el-descriptions title="资源信息" :column="2" border>
<el-descriptions-item label="类型">
<el-tag size="small">{{ EnumValue.getLabelByValue(ResourceTypeEnum, infoDialog.data.type) }}</el-tag>
</el-descriptions-item>
<el-descriptions-item label="名称">{{ infoDialog.data.name }}</el-descriptions-item>
<el-descriptions-item label="code[菜单path]">{{ infoDialog.data.code }}</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="图标">
<SvgIcon :name="infoDialog.data.meta.icon" />
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="路由名">
{{ infoDialog.data.meta.routeName }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="组件路径">
{{ infoDialog.data.meta.component }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="是否缓存">
{{ infoDialog.data.meta.isKeepAlive ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="是否隐藏">
{{ infoDialog.data.meta.isHide ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="tag不可删除">
{{ infoDialog.data.meta.isAffix ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue" label="外链">
{{ infoDialog.data.meta.linkType ? '是' : '否' }}
</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.type == menuTypeValue && infoDialog.data.meta.linkType > 0" label="外链">
{{ infoDialog.data.meta.link }}
</el-descriptions-item>
<el-descriptions-item label="创建者">{{ infoDialog.data.creator }}</el-descriptions-item>
<el-descriptions-item label="创建时间">{{ dateFormat(infoDialog.data.createTime) }} </el-descriptions-item>
<el-descriptions-item label="修改者">{{ infoDialog.data.modifier }}</el-descriptions-item>
<el-descriptions-item label="更新时间">{{ dateFormat(infoDialog.data.updateTime) }} </el-descriptions-item>
</el-descriptions>
</el-dialog>
<contextmenu :dropdown="state.contextmenu.dropdown" :items="state.contextmenu.items" ref="contextmenuRef" />
</div>
</template>
@@ -108,8 +120,9 @@ import ResourceEdit from './ResourceEdit.vue';
import { ResourceTypeEnum } from '../enums';
import { resourceApi } from '../api';
import { dateFormat } from '@/common/utils/date';
import EnumValue from '@/common/Enum';
import EnumTag from '@/components/enumtag/EnumTag.vue';
import { Contextmenu, ContextmenuItem } from '@/components/contextmenu';
import { Splitpanes, Pane } from 'splitpanes';
const menuTypeValue = ResourceTypeEnum.Menu.value;
const permissionTypeValue = ResourceTypeEnum.Permission.value;
@@ -130,7 +143,7 @@ const contextmenuRef = ref();
const filterResource = ref();
const resourceTreeRef = ref();
const contextmenuInfo = new ContextmenuItem('info', '详情').withIcon('View').withOnClick((data: any) => info(data));
const ResourceDetail = 'resourceDetail';
const contextmenuAdd = new ContextmenuItem('add', '添加子资源')
.withIcon('circle-plus')
@@ -166,7 +179,7 @@ const state = reactive({
x: 0,
y: 0,
},
items: [contextmenuInfo, contextmenuAdd, contextmenuEdit, contextmenuEnable, contextmenuDisable, contextmenuDel],
items: [contextmenuAdd, contextmenuEdit, contextmenuEnable, contextmenuDisable, contextmenuDel],
},
//弹出框对象
dialogForm: {
@@ -177,29 +190,15 @@ const state = reactive({
// 资源类型选择是否选
typeDisabled: true,
},
//资源信息弹出框对象
infoDialog: {
title: '',
visible: false,
// 资源类型选择是否选
data: {
meta: {} as any,
name: '',
type: null,
creator: '',
modifier: '',
createTime: '',
updateTime: '',
code: '',
},
},
data: [],
// 展开的节点
defaultExpandedKeys: [] as any[],
activeTabName: ResourceDetail,
currentResource: null as any,
});
const { dialogForm, infoDialog, data, defaultExpandedKeys } = toRefs(state);
const { currentResource, dialogForm, data, defaultExpandedKeys } = toRefs(state);
onMounted(() => {
search();
@@ -229,9 +228,15 @@ const nodeContextmenu = (event: any, data: any) => {
contextmenuRef.value.openContextmenu(data);
};
const treeNodeClick = () => {
const treeNodeClick = async (data: any) => {
// 关闭可能存在的右击菜单
contextmenuRef.value.closeContextmenu();
let info = await resourceApi.detail.request({ id: data.id });
state.currentResource = info;
if (info.meta && info.meta != '') {
state.currentResource.meta = JSON.parse(info.meta);
}
};
const deleteMenu = (data: any) => {
@@ -390,15 +395,6 @@ const removeDeafultExpandId = (id: any) => {
state.defaultExpandedKeys.splice(index, 1);
}
};
const info = async (data: any) => {
let info = await resourceApi.detail.request({ id: data.id });
state.infoDialog.data = info;
if (info.meta && info.meta != '') {
state.infoDialog.data.meta = JSON.parse(info.meta);
}
state.infoDialog.visible = true;
};
</script>
<style lang="scss">
.system-resouce-list {
@@ -410,6 +406,11 @@ const info = async (data: any) => {
.tree-data {
height: calc(100vh - 202px);
}
.el-tree {
display: inline-block;
min-width: 100%;
}
}
.none-select {

150
server/internal/db/api/db.go
View File

@@ -17,7 +17,6 @@ import (
tagapp "mayfly-go/internal/tag/application"
tagentity "mayfly-go/internal/tag/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/model"
"mayfly-go/pkg/req"
@@ -25,12 +24,11 @@ import (
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"mayfly-go/pkg/ws"
"sort"
"strconv"
"strings"
"time"
"github.com/kanzihuang/vitess/go/vt/sqlparser"
"github.com/may-fly/cast"
)
type Db struct {
@@ -81,9 +79,8 @@ func (d *Db) DeleteDb(rc *req.Ctx) {
ctx := rc.MetaCtx
for _, v := range ids {
value, err := strconv.Atoi(v)
biz.ErrIsNilAppendErr(err, "string类型转换为int异常: %s")
dbId := uint64(value)
dbId := cast.ToUint64(v)
biz.NotBlank(dbId, "存在错误dbId")
d.DbApp.Delete(ctx, dbId)
// 删除该库的sql执行记录
d.DbSqlExecApp.DeleteBy(ctx, &entity.DbSqlExec{DbId: dbId})
@@ -118,7 +115,7 @@ func (d *Db) ExecSql(rc *req.Ctx) {
ctx, cancel := context.WithTimeout(rc.MetaCtx, time.Duration(config.GetDbms().SqlExecTl)*time.Second)
defer cancel()
sqls, err := sqlparser.SplitStatementToPieces(sql, sqlparser.WithDialect(dbConn.GetMetaData().SqlParserDialect()))
sqls, err := sqlparser.SplitStatementToPieces(sql, sqlparser.WithDialect(dbConn.GetMetaData().GetSqlParserDialect()))
biz.ErrIsNil(err, "SQL解析错误,请检查您的执行SQL")
isMulti := len(sqls) > 1
var execResAll *application.DbSqlExecRes
@@ -199,7 +196,7 @@ func (d *Db) ExecSqlFile(rc *req.Ctx) {
var sql string
tokenizer := sqlparser.NewReaderTokenizer(file,
sqlparser.WithCacheInBuffer(), sqlparser.WithDialect(dbConn.GetMetaData().SqlParserDialect()))
sqlparser.WithCacheInBuffer(), sqlparser.WithDialect(dbConn.GetMetaData().GetSqlParserDialect()))
executedStatements := 0
progressId := stringx.Rand(32)
@@ -264,7 +261,7 @@ func (d *Db) ExecSqlFile(rc *req.Ctx) {
// 数据库dump
func (d *Db) DumpSql(rc *req.Ctx) {
dbId := getDbId(rc)
dbNamesStr := rc.Query("db")
dbName := rc.Query("db")
dumpType := rc.Query("type")
tablesStr := rc.Query("tables")
extName := rc.Query("extName")
@@ -286,146 +283,37 @@ func (d *Db) DumpSql(rc *req.Ctx) {
biz.ErrIsNilAppendErr(d.TagApp.CanAccess(la.Id, d.TagApp.ListTagPathByResource(consts.TagResourceTypeDb, db.Code)...), "%s")
now := time.Now()
filename := fmt.Sprintf("%s.%s.sql%s", db.Name, now.Format("20060102150405"), extName)
filename := fmt.Sprintf("%s-%s.%s.sql%s", db.Name, dbName, now.Format("20060102150405"), extName)
rc.Header("Content-Type", "application/octet-stream")
rc.Header("Content-Disposition", "attachment; filename="+filename)
if extName != ".gz" {
rc.Header("Content-Encoding", "gzip")
}
var dbNames, tables []string
if len(dbNamesStr) > 0 {
dbNames = strings.Split(dbNamesStr, ",")
}
if len(dbNames) == 1 && len(tablesStr) > 0 {
var tables []string
if len(tablesStr) > 0 {
tables = strings.Split(tablesStr, ",")
}
writer := newGzipWriter(rc.GetWriter())
defer func() {
msg := anyx.ToString(recover())
if len(msg) > 0 {
msg = "数据库导出失败: " + msg
writer.WriteString(msg)
rc.GetWriter().Write([]byte(msg))
d.MsgApp.CreateAndSend(la, msgdto.ErrSysMsg("数据库导出失败", msg))
}
writer.Close()
}()
for _, dbName := range dbNames {
d.dumpDb(rc.MetaCtx, writer, dbId, dbName, tables, needStruct, needData)
}
biz.ErrIsNil(d.DbApp.DumpDb(rc.MetaCtx, &application.DumpDbReq{
DbId: dbId,
DbName: dbName,
Tables: tables,
DumpDDL: needStruct,
DumpData: needData,
Writer: rc.GetWriter(),
}))
rc.ReqParam = collx.Kvs("db", db, "databases", dbNamesStr, "tables", tablesStr, "dumpType", dumpType)
}
func (d *Db) dumpDb(ctx context.Context, writer *gzipWriter, dbId uint64, dbName string, tables []string, needStruct bool, needData bool) {
dbConn, err := d.DbApp.GetDbConn(dbId, dbName)
biz.ErrIsNil(err)
writer.WriteString("\n-- ----------------------------")
writer.WriteString("\n-- 导出平台: mayfly-go")
writer.WriteString(fmt.Sprintf("\n-- 导出时间: %s ", time.Now().Format("2006-01-02 15:04:05")))
writer.WriteString(fmt.Sprintf("\n-- 导出数据库: %s ", dbName))
writer.WriteString("\n-- ----------------------------\n\n")
dbMeta := dbConn.GetMetaData()
if len(tables) == 0 {
ti, err := dbMeta.GetTables()
biz.ErrIsNil(err)
tables = make([]string, len(ti))
for i, table := range ti {
tables[i] = table.TableName
}
}
// 查询列信息，后面生成建表ddl和insert都需要列信息
columns, err := dbMeta.GetColumns(tables...)
// 以表名分组，存放每个表的列信息
columnMap := make(map[string][]dbi.Column)
for _, column := range columns {
columnMap[column.TableName] = append(columnMap[column.TableName], column)
}
// 按表名排序
sort.Strings(tables)
quoteSchema := dbMeta.QuoteIdentifier(dbConn.Info.CurrentSchema())
// 遍历获取每个表的信息
for _, tableName := range tables {
quoteTableName := dbMeta.QuoteIdentifier(tableName)
writer.TryFlush()
// 查询表信息，主要是为了查询表注释
tbs, err := dbMeta.GetTables(tableName)
biz.ErrIsNil(err)
if err != nil || tbs == nil || len(tbs) <= 0 {
panic(errorx.NewBiz(fmt.Sprintf("获取表信息失败：%s", tableName)))
}
tabInfo := dbi.Table{
TableName: tableName,
TableComment: tbs[0].TableComment,
}
// 生成表结构信息
if needStruct {
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表结构: %s \n-- ----------------------------\n", tableName))
tbDdlArr := dbMeta.GenerateTableDDL(columnMap[tableName], tabInfo, true)
for _, ddl := range tbDdlArr {
writer.WriteString(ddl + ";\n")
}
}
// 生成insert sql，数据在索引前，加速insert
if needData {
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表记录: %s \n-- ----------------------------\n", tableName))
dbMeta.BeforeDumpInsert(writer, quoteTableName)
// 获取列信息
quoteColNames := make([]string, 0)
for _, col := range columnMap[tableName] {
quoteColNames = append(quoteColNames, dbMeta.QuoteIdentifier(col.ColumnName))
}
converter := dbMeta.GetDataConverter()
_ = dbConn.WalkTableRows(context.Background(), quoteTableName, func(row map[string]any, _ []*dbi.QueryColumn) error {
rowValues := make([]string, len(columnMap[tableName]))
for i, col := range columnMap[tableName] {
rowValues[i] = converter.WrapValue(row[col.ColumnName], converter.GetDataType(string(col.DataType)))
}
beforeInsert := dbMeta.BeforeDumpInsertSql(quoteSchema, quoteTableName)
insertSQL := fmt.Sprintf("%s INSERT INTO %s (%s) values(%s)", beforeInsert, quoteTableName, strings.Join(quoteColNames, ", "), strings.Join(rowValues, ", "))
writer.WriteString(insertSQL + ";\n")
return nil
})
dbMeta.AfterDumpInsert(writer, tableName, columnMap[tableName])
}
indexs, err := dbMeta.GetTableIndex(tableName)
biz.ErrIsNil(err)
// 过滤主键索引
idxs := make([]dbi.Index, 0)
for _, idx := range indexs {
if !idx.IsPrimaryKey {
idxs = append(idxs, idx)
}
}
if len(idxs) > 0 {
// 最后添加索引
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表索引: %s \n-- ----------------------------\n", tableName))
sqlArr := dbMeta.GenerateIndexDDL(idxs, tabInfo)
for _, sqlStr := range sqlArr {
writer.WriteString(sqlStr + ";\n")
}
}
}
rc.ReqParam = collx.Kvs("db", db, "database", dbName, "tables", tablesStr, "dumpType", dumpType)
}
func (d *Db) TableInfos(rc *req.Ctx) {

127
server/internal/db/application/db.go
View File

@@ -2,6 +2,7 @@ package application
import (
"context"
"fmt"
"mayfly-go/internal/common/consts"
"mayfly-go/internal/db/dbm"
"mayfly-go/internal/db/dbm/dbi"
@@ -9,12 +10,15 @@ import (
"mayfly-go/internal/db/domain/repository"
tagapp "mayfly-go/internal/tag/application"
"mayfly-go/pkg/base"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/model"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"mayfly-go/pkg/utils/structx"
"sort"
"strings"
"time"
)
type Db interface {
@@ -38,6 +42,9 @@ type Db interface {
// 根据数据库实例id获取连接，随机返回该instanceId下已连接的conn，若不存在则是使用该instanceId关联的db进行连接并返回。
GetDbConnByInstanceId(instanceId uint64) (*dbi.DbConn, error)
// DumpDb dumpDb
DumpDb(ctx context.Context, reqParam *DumpDbReq) error
}
type dbAppImpl struct {
@@ -189,6 +196,126 @@ func (d *dbAppImpl) GetDbConnByInstanceId(instanceId uint64) (*dbi.DbConn, error
return d.GetDbConn(firstDb.Id, strings.Split(firstDb.Database, " ")[0])
}
func (d *dbAppImpl) DumpDb(ctx context.Context, reqParam *DumpDbReq) error {
writer := newGzipWriter(reqParam.Writer)
defer writer.Close()
dbId := reqParam.DbId
dbName := reqParam.DbName
tables := reqParam.Tables
dbConn, err := d.GetDbConn(dbId, dbName)
if err != nil {
return err
}
writer.WriteString("\n-- ----------------------------")
writer.WriteString("\n-- 导出平台: mayfly-go")
writer.WriteString(fmt.Sprintf("\n-- 导出时间: %s ", time.Now().Format("2006-01-02 15:04:05")))
writer.WriteString(fmt.Sprintf("\n-- 导出数据库: %s ", dbName))
writer.WriteString("\n-- ----------------------------\n\n")
dbMeta := dbConn.GetMetaData()
if len(tables) == 0 {
ti, err := dbMeta.GetTables()
biz.ErrIsNil(err)
tables = make([]string, len(ti))
for i, table := range ti {
tables[i] = table.TableName
}
}
// 查询列信息，后面生成建表ddl和insert都需要列信息
columns, err := dbMeta.GetColumns(tables...)
biz.ErrIsNil(err)
// 以表名分组，存放每个表的列信息
columnMap := make(map[string][]dbi.Column)
for _, column := range columns {
columnMap[column.TableName] = append(columnMap[column.TableName], column)
}
// 按表名排序
sort.Strings(tables)
quoteSchema := dbMeta.QuoteIdentifier(dbConn.Info.CurrentSchema())
dumpHelper := dbMeta.GetDumpHelper()
dataHelper := dbMeta.GetDataHelper()
// 遍历获取每个表的信息
for _, tableName := range tables {
quoteTableName := dbMeta.QuoteIdentifier(tableName)
writer.TryFlush()
// 查询表信息，主要是为了查询表注释
tbs, err := dbMeta.GetTables(tableName)
biz.ErrIsNil(err)
if err != nil || tbs == nil || len(tbs) <= 0 {
panic(errorx.NewBiz(fmt.Sprintf("获取表信息失败：%s", tableName)))
}
tabInfo := dbi.Table{
TableName: tableName,
TableComment: tbs[0].TableComment,
}
// 生成表结构信息
if reqParam.DumpDDL {
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表结构: %s \n-- ----------------------------\n", tableName))
tbDdlArr := dbMeta.GenerateTableDDL(columnMap[tableName], tabInfo, true)
for _, ddl := range tbDdlArr {
writer.WriteString(ddl + ";\n")
}
}
// 生成insert sql，数据在索引前，加速insert
if reqParam.DumpData {
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表记录: %s \n-- ----------------------------\n", tableName))
dumpHelper.BeforeInsert(writer, quoteTableName)
// 获取列信息
quoteColNames := make([]string, 0)
for _, col := range columnMap[tableName] {
quoteColNames = append(quoteColNames, dbMeta.QuoteIdentifier(col.ColumnName))
}
_ = dbConn.WalkTableRows(ctx, quoteTableName, func(row map[string]any, _ []*dbi.QueryColumn) error {
rowValues := make([]string, len(columnMap[tableName]))
for i, col := range columnMap[tableName] {
rowValues[i] = dataHelper.WrapValue(row[col.ColumnName], dataHelper.GetDataType(string(col.DataType)))
}
beforeInsert := dumpHelper.BeforeInsertSql(quoteSchema, quoteTableName)
insertSQL := fmt.Sprintf("%s INSERT INTO %s (%s) values(%s)", beforeInsert, quoteTableName, strings.Join(quoteColNames, ", "), strings.Join(rowValues, ", "))
writer.WriteString(insertSQL + ";\n")
return nil
})
dumpHelper.AfterInsert(writer, tableName, columnMap[tableName])
}
indexs, err := dbMeta.GetTableIndex(tableName)
biz.ErrIsNil(err)
// 过滤主键索引
idxs := make([]dbi.Index, 0)
for _, idx := range indexs {
if !idx.IsPrimaryKey {
idxs = append(idxs, idx)
}
}
if len(idxs) > 0 {
// 最后添加索引
writer.WriteString(fmt.Sprintf("\n-- ----------------------------\n-- 表索引: %s \n-- ----------------------------\n", tableName))
sqlArr := dbMeta.GenerateIndexDDL(idxs, tabInfo)
for _, sqlStr := range sqlArr {
writer.WriteString(sqlStr + ";\n")
}
}
}
return nil
}
func toDbInfo(instance *entity.DbInstance, dbId uint64, database string, tagPath ...string) *dbi.DbInfo {
di := new(dbi.DbInfo)
di.InstanceId = instance.Id

10
server/internal/db/application/db_data_sync.go
View File

@@ -163,7 +163,7 @@ func (app *dataSyncAppImpl) RunCronJob(id uint64) error {
} else {
updFieldValType = dbi.DataTypeNumber
}
wrapUpdFieldVal := srcConn.GetMetaData().GetDataConverter().WrapValue(task.UpdFieldVal, updFieldValType)
wrapUpdFieldVal := srcConn.GetMetaData().GetDataHelper().WrapValue(task.UpdFieldVal, updFieldValType)
updSql = fmt.Sprintf("and %s > %s", task.UpdField, wrapUpdFieldVal)
orderSql = "order by " + task.UpdField + " asc "
@@ -249,7 +249,7 @@ func (app *dataSyncAppImpl) doDataSync(sql string, task *entity.DataSyncTask) (*
updFieldType = dbi.DataTypeString
for _, column := range columns {
if strings.EqualFold(column.Name, updFieldName) {
updFieldType = srcMetaData.GetDataConverter().GetDataType(column.Type)
updFieldType = srcMetaData.GetDataHelper().GetDataType(column.Type)
break
}
}
@@ -332,7 +332,7 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
updFieldVal = srcRes[len(srcRes)-1][strings.ToLower(updFieldName)]
}
task.UpdFieldVal = srcMetaData.GetDataConverter().FormatData(updFieldVal, updFieldType)
task.UpdFieldVal = srcMetaData.GetDataHelper().FormatData(updFieldVal, updFieldType)
// 获取目标库字段数组
targetWrapColumns := make([]string, 0)
@@ -343,7 +343,7 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
for _, item := range fieldMap {
targetField := item["target"]
srcField := item["target"]
srcFieldTypes[srcField] = srcMetaData.GetDataConverter().GetDataType(srcColumnTypes[item["src"]])
srcFieldTypes[srcField] = srcMetaData.GetDataHelper().GetDataType(srcColumnTypes[item["src"]])
targetWrapColumns = append(targetWrapColumns, targetMetaData.QuoteIdentifier(targetField))
srcColumns = append(srcColumns, srcField)
}
@@ -354,7 +354,7 @@ func (app *dataSyncAppImpl) srcData2TargetDb(srcRes []map[string]any, fieldMap [
rawValue := make([]any, 0)
for _, column := range srcColumns {
// 某些情况，如oracle，需要转换时间类型的字符串为time类型
res := srcMetaData.GetDataConverter().ParseData(record[column], srcFieldTypes[column])
res := srcMetaData.GetDataHelper().ParseData(record[column], srcFieldTypes[column])
rawValue = append(rawValue, res)
}
values = append(values, rawValue)

12
server/internal/db/application/db_transfer.go
View File

@@ -148,9 +148,9 @@ func (app *dbTransferAppImpl) transferTables(task *entity.DbTransferTask, srcCon
end("没有需要迁移的表", nil)
return
}
srcMeta := srcConn.GetMetaData()
// 查询源表列信息
columns, err := srcConn.GetMetaData().GetColumns(tableNames...)
columns, err := srcMeta.GetColumns(tableNames...)
if err != nil {
end("获取源表列信息失败", err)
return
@@ -168,15 +168,17 @@ func (app *dbTransferAppImpl) transferTables(task *entity.DbTransferTask, srcCon
ctx := context.Background()
srcColumnHelper := srcMeta.GetColumnHelper()
targetColumnHelper := targetConn.GetMetaData().GetColumnHelper()
for _, tbName := range sortTableNames {
cols := columnMap[tbName]
targetCols := make([]dbi.Column, 0)
for _, col := range cols {
colPtr := &col
// 源库列转为公共列
srcDialect.ToCommonColumn(colPtr)
srcColumnHelper.ToCommonColumn(colPtr)
// 公共列转为目标库列
targetDialect.ToColumn(colPtr)
targetColumnHelper.ToColumn(colPtr)
targetCols = append(targetCols, *colPtr)
}
@@ -230,7 +232,7 @@ func (app *dbTransferAppImpl) transferData(ctx context.Context, tableName string
batchSize := 1000 // 每次查询并迁移1000条数据
var err error
srcMeta := srcConn.GetMetaData()
srcConverter := srcMeta.GetDataConverter()
srcConverter := srcMeta.GetDataHelper()
// 游标查询源表数据，并批量插入目标表
err = srcConn.WalkTableRows(ctx, tableName, func(row map[string]any, columns []*dbi.QueryColumn) error {

2
server/internal/db/api/gzip_writer.go → server/internal/db/application/gzip_writer.go
View File

@@ -1,4 +1,4 @@
package api
package application
import (
"compress/gzip"

13
server/internal/db/application/param.go Normal file
View File

@@ -0,0 +1,13 @@
package application
import "io"
type DumpDbReq struct {
DbId uint64
DbName string
Tables []string
DumpDDL bool // 是否dump ddl
DumpData bool // 是否dump data
Writer io.Writer
}

18
server/internal/db/dbm/dbi/dialect.go
View File

@@ -40,12 +40,6 @@ type Dialect interface {
// 有些数据库迁移完数据之后，需要更新表自增序列为当前表最大值
UpdateSequence(tableName string, columns []Column)
// 数据库方言自带的列转换为公共列
ToCommonColumn(dialectColumn *Column)
// 公共列转为各个数据库方言自带的列
ToColumn(commonColumn *Column)
}
type DefaultDialect struct {
@@ -56,14 +50,4 @@ func (dd *DefaultDialect) GetDbProgram() (DbProgram, error) {
return nil, errors.New("not support db program")
}
func (dd *DefaultDialect) ToCommonColumn(dialectColumn *Column) {
}
func (dd *DefaultDialect) ToColumn(commonColumn *Column) {
}
func (dd *DefaultDialect) UpdateSequence(tableName string, columns []Column) {
}
func (dd *DefaultDialect) UpdateSequence(tableName string, columns []Column) {}

10
server/internal/db/dbm/dbi/metadata.go
View File

@@ -26,7 +26,7 @@ type MetaData interface {
GetColumns(tableNames ...string) ([]Column, error)
// 根据数据库类型修复字段长度、精度等
FixColumn(column *Column)
// FixColumn(column *Column)
// 获取表主键字段名，没有主键标识则默认第一个字段
GetPrimaryKey(tableName string) (string, error)
@@ -43,8 +43,8 @@ type MetaData interface {
GetSchemas() ([]string, error)
// 获取数据转换器用于解析格式化列数据等
GetDataConverter() DataConverter
// 获取数据处理助手 用于解析格式化列数据等
GetDataHelper() DataHelper
}
// GenerateSQLStepFunc 生成insert sql的step函数，用于生成insert sql时，每生成100条sql时调用
@@ -148,8 +148,8 @@ const (
DataTypeDateTime DataType = "datetime"
)
// 数据转换器
type DataConverter interface {
// 列数据处理帮助方法
type DataHelper interface {
// 获取数据对应的类型
// @param dbColumnType 数据库原始列类型，如varchar等
GetDataType(dbColumnType string) DataType

66
server/internal/db/dbm/dbi/metadata_base.go
View File

@@ -1,10 +1,11 @@
package dbi
import (
pq "gitee.com/liuzongyang/libpq"
"github.com/kanzihuang/vitess/go/vt/sqlparser"
"io"
"strings"
pq "gitee.com/liuzongyang/libpq"
"github.com/kanzihuang/vitess/go/vt/sqlparser"
)
type BaseMetaData interface {
@@ -29,13 +30,13 @@ type BaseMetaData interface {
// replaced by two backslashes (i.e. "\\") and the C-style escape identifier
QuoteLiteral(literal string) string
SqlParserDialect() sqlparser.Dialect
GetSqlParserDialect() sqlparser.Dialect
BeforeDumpInsert(writer io.Writer, tableName string)
// GetColumnHelper
GetColumnHelper() ColumnHelper
BeforeDumpInsertSql(quoteSchema string, quoteTableName string) string
AfterDumpInsert(writer io.Writer, tableName string, columns []Column)
// GetDumpHeler
GetDumpHelper() DumpHelper
}
// 默认实现，若需要覆盖，则由各个数据库MetaData实现去覆盖重写
@@ -58,16 +59,59 @@ func (dd *DefaultMetaData) QuoteLiteral(literal string) string {
return pq.QuoteLiteral(literal)
}
func (dd *DefaultMetaData) SqlParserDialect() sqlparser.Dialect {
func (dd *DefaultMetaData) GetSqlParserDialect() sqlparser.Dialect {
return sqlparser.PostgresDialect{}
}
func (dd *DefaultMetaData) BeforeDumpInsert(writer io.Writer, tableName string) {
func (dd *DefaultMetaData) GetDumpHelper() DumpHelper {
return new(DefaultDumpHelper)
}
func (dd *DefaultMetaData) GetColumnHelper() ColumnHelper {
return new(DefaultColumnHelper)
}
// ColumnHelper 数据库迁移辅助方法
type ColumnHelper interface {
// ToCommonColumn 数据库方言自带的列转换为公共列
ToCommonColumn(dialectColumn *Column)
// ToColumn 公共列转为各个数据库方言自带的列
ToColumn(commonColumn *Column)
// FixColumn 根据数据库类型修复字段长度、精度等
FixColumn(column *Column)
}
type DefaultColumnHelper struct {
}
func (dd *DefaultColumnHelper) ToCommonColumn(dialectColumn *Column) {}
func (dd *DefaultColumnHelper) ToColumn(commonColumn *Column) {}
func (dd *DefaultColumnHelper) FixColumn(column *Column) {}
// DumpHelper 导出辅助方法
type DumpHelper interface {
BeforeInsert(writer io.Writer, tableName string)
BeforeInsertSql(quoteSchema string, quoteTableName string) string
AfterInsert(writer io.Writer, tableName string, columns []Column)
}
type DefaultDumpHelper struct {
}
func (dd *DefaultDumpHelper) BeforeInsert(writer io.Writer, tableName string) {
writer.Write([]byte("BEGIN;\n"))
}
func (dd *DefaultMetaData) BeforeDumpInsertSql(quoteSchema string, tableName string) string {
func (dd *DefaultDumpHelper) BeforeInsertSql(quoteSchema string, quoteTableName string) string {
return ""
}
func (dd *DefaultMetaData) AfterDumpInsert(writer io.Writer, tableName string, columns []Column) {
func (dd *DefaultDumpHelper) AfterInsert(writer io.Writer, tableName string, columns []Column) {
writer.Write([]byte("COMMIT;\n"))
}

27
server/internal/db/dbm/dm/dialect.go
View File

@@ -138,7 +138,7 @@ func (dd *DMDialect) CopyTable(copy *dbi.DbCopyTable) error {
ddl = strings.ReplaceAll(ddl, fmt.Sprintf("\"%s\"", strings.ToUpper(tableName)), fmt.Sprintf("\"%s\"", strings.ToUpper(newTableName)))
// 去除空格换行
ddl = stringx.TrimSpaceAndBr(ddl)
sqls, err := sqlparser.SplitStatementToPieces(ddl, sqlparser.WithDialect(dd.dc.GetMetaData().SqlParserDialect()))
sqls, err := sqlparser.SplitStatementToPieces(ddl, sqlparser.WithDialect(dd.dc.GetMetaData().GetSqlParserDialect()))
for _, sql := range sqls {
_, _ = dd.dc.Exec(sql)
}
@@ -163,31 +163,6 @@ func (dd *DMDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (dd *DMDialect) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (dd *DMDialect) ToColumn(commonColumn *dbi.Column) {
ctype := dmColumnTypeMap[commonColumn.DataType]
meta := dd.dc.GetMetaData()
if ctype == "" {
commonColumn.DataType = "VARCHAR"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
meta.FixColumn(commonColumn)
}
}
func (dd *DMDialect) CreateTable(columns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
sqlArr := dd.dc.GetMetaData().GenerateTableDDL(columns, tableInfo, dropOldTable)

222
server/internal/db/dbm/dm/helper.go Normal file
View File

@@ -0,0 +1,222 @@
package dm
import (
"fmt"
"io"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
"strings"
"time"
)
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
// 达梦数据类型 对应 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"CHAR": dbi.CommonTypeChar, // 字符数据类型
"VARCHAR": dbi.CommonTypeVarchar,
"TEXT": dbi.CommonTypeText,
"LONG": dbi.CommonTypeText,
"LONGVARCHAR": dbi.CommonTypeLongtext,
"IMAGE": dbi.CommonTypeLongtext,
"LONGVARBINARY": dbi.CommonTypeLongtext,
"BLOB": dbi.CommonTypeBlob,
"CLOB": dbi.CommonTypeText,
"NUMERIC": dbi.CommonTypeNumber, // 精确数值数据类型
"DECIMAL": dbi.CommonTypeNumber,
"NUMBER": dbi.CommonTypeNumber,
"INTEGER": dbi.CommonTypeInt,
"INT": dbi.CommonTypeInt,
"BIGINT": dbi.CommonTypeBigint,
"TINYINT": dbi.CommonTypeTinyint,
"BYTE": dbi.CommonTypeTinyint,
"SMALLINT": dbi.CommonTypeSmallint,
"BIT": dbi.CommonTypeTinyint,
"DOUBLE": dbi.CommonTypeNumber, // 近似数值类型
"FLOAT": dbi.CommonTypeNumber,
"DATE": dbi.CommonTypeDate, // 一般日期时间数据类型
"TIME": dbi.CommonTypeTime,
"TIMESTAMP": dbi.CommonTypeTimestamp,
}
// 公共数据类型 对应 达梦数据类型
dmColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "VARCHAR",
dbi.CommonTypeChar: "CHAR",
dbi.CommonTypeText: "TEXT",
dbi.CommonTypeBlob: "BLOB",
dbi.CommonTypeLongblob: "TEXT",
dbi.CommonTypeLongtext: "TEXT",
dbi.CommonTypeBinary: "TEXT",
dbi.CommonTypeMediumblob: "TEXT",
dbi.CommonTypeMediumtext: "TEXT",
dbi.CommonTypeVarbinary: "TEXT",
dbi.CommonTypeInt: "INT",
dbi.CommonTypeSmallint: "SMALLINT",
dbi.CommonTypeTinyint: "TINYINT",
dbi.CommonTypeNumber: "NUMBER",
dbi.CommonTypeBigint: "BIGINT",
dbi.CommonTypeDatetime: "TIMESTAMP",
dbi.CommonTypeDate: "DATE",
dbi.CommonTypeTime: "DATE",
dbi.CommonTypeTimestamp: "TIMESTAMP",
dbi.CommonTypeEnum: "TEXT",
dbi.CommonTypeJSON: "TEXT",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:08:22.275688+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return str
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (ch *ColumnHelper) ToColumn(commonColumn *dbi.Column) {
ctype := dmColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "VARCHAR"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
ch.FixColumn(commonColumn)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
// 如果是date，不设长度
if collx.ArrayAnyMatches([]string{"date", "time"}, strings.ToLower(string(column.DataType))) {
column.CharMaxLength = 0
column.NumPrecision = 0
} else
// 如果是char且长度未设置，则默认长度2000
if collx.ArrayAnyMatches([]string{"char"}, strings.ToLower(string(column.DataType))) && column.CharMaxLength == 0 {
column.CharMaxLength = 2000
}
}
type DumpHelper struct {
}
func (dh *DumpHelper) BeforeInsert(writer io.Writer, tableName string) {
}
func (dh *DumpHelper) BeforeInsertSql(quoteSchema string, tableName string) string {
return fmt.Sprintf("set identity_insert %s on;", tableName)
}
func (dh *DumpHelper) AfterInsert(writer io.Writer, tableName string, columns []dbi.Column) {
writer.Write([]byte("COMMIT;\n"))
}

192
server/internal/db/dbm/dm/metadata.go
View File

@@ -2,16 +2,13 @@ package dm
import (
"fmt"
"io"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
@@ -98,6 +95,7 @@ func (dd *DMMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error) {
return nil, err
}
columnHelper := dd.dc.GetMetaData().GetColumnHelper()
columns := make([]dbi.Column, 0)
for _, re := range res {
column := dbi.Column{
@@ -113,24 +111,12 @@ func (dd *DMMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error) {
NumPrecision: cast.ToInt(re["NUM_PRECISION"]),
NumScale: cast.ToInt(re["NUM_SCALE"]),
}
dd.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
return columns, nil
}
func (dd *DMMetaData) FixColumn(column *dbi.Column) {
// 如果是date，不设长度
if collx.ArrayAnyMatches([]string{"date", "time"}, strings.ToLower(string(column.DataType))) {
column.CharMaxLength = 0
column.NumPrecision = 0
} else
// 如果是char且长度未设置，则默认长度2000
if collx.ArrayAnyMatches([]string{"char"}, strings.ToLower(string(column.DataType))) && column.CharMaxLength == 0 {
column.CharMaxLength = 2000
}
}
func (dd *DMMetaData) GetPrimaryKey(tablename string) (string, error) {
columns, err := dd.GetColumns(tablename)
if err != nil {
@@ -340,176 +326,14 @@ func (dd *DMMetaData) GetSchemas() ([]string, error) {
return schemaNames, nil
}
func (dd *DMMetaData) BeforeDumpInsert(writer io.Writer, tableName string) {
func (dd *DMMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
func (dd *DMMetaData) BeforeDumpInsertSql(quoteSchema string, tableName string) string {
return fmt.Sprintf("set identity_insert %s on;", tableName)
func (dd *DMMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}
func (dd *DMMetaData) AfterDumpInsert(writer io.Writer, tableName string, columns []dbi.Column) {
writer.Write([]byte("COMMIT;\n"))
}
func (dd *DMMetaData) GetDataConverter() dbi.DataConverter {
return converter
}
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
// 达梦数据类型 对应 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"CHAR": dbi.CommonTypeChar, // 字符数据类型
"VARCHAR": dbi.CommonTypeVarchar,
"TEXT": dbi.CommonTypeText,
"LONG": dbi.CommonTypeText,
"LONGVARCHAR": dbi.CommonTypeLongtext,
"IMAGE": dbi.CommonTypeLongtext,
"LONGVARBINARY": dbi.CommonTypeLongtext,
"BLOB": dbi.CommonTypeBlob,
"CLOB": dbi.CommonTypeText,
"NUMERIC": dbi.CommonTypeNumber, // 精确数值数据类型
"DECIMAL": dbi.CommonTypeNumber,
"NUMBER": dbi.CommonTypeNumber,
"INTEGER": dbi.CommonTypeInt,
"INT": dbi.CommonTypeInt,
"BIGINT": dbi.CommonTypeBigint,
"TINYINT": dbi.CommonTypeTinyint,
"BYTE": dbi.CommonTypeTinyint,
"SMALLINT": dbi.CommonTypeSmallint,
"BIT": dbi.CommonTypeTinyint,
"DOUBLE": dbi.CommonTypeNumber, // 近似数值类型
"FLOAT": dbi.CommonTypeNumber,
"DATE": dbi.CommonTypeDate, // 一般日期时间数据类型
"TIME": dbi.CommonTypeTime,
"TIMESTAMP": dbi.CommonTypeTimestamp,
}
// 公共数据类型 对应 达梦数据类型
dmColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "VARCHAR",
dbi.CommonTypeChar: "CHAR",
dbi.CommonTypeText: "TEXT",
dbi.CommonTypeBlob: "BLOB",
dbi.CommonTypeLongblob: "TEXT",
dbi.CommonTypeLongtext: "TEXT",
dbi.CommonTypeBinary: "TEXT",
dbi.CommonTypeMediumblob: "TEXT",
dbi.CommonTypeMediumtext: "TEXT",
dbi.CommonTypeVarbinary: "TEXT",
dbi.CommonTypeInt: "INT",
dbi.CommonTypeSmallint: "SMALLINT",
dbi.CommonTypeTinyint: "TINYINT",
dbi.CommonTypeNumber: "NUMBER",
dbi.CommonTypeBigint: "BIGINT",
dbi.CommonTypeDatetime: "TIMESTAMP",
dbi.CommonTypeDate: "DATE",
dbi.CommonTypeTime: "DATE",
dbi.CommonTypeTimestamp: "TIMESTAMP",
dbi.CommonTypeEnum: "TEXT",
dbi.CommonTypeJSON: "TEXT",
}
)
type DataConverter struct {
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:08:22.275688+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return str
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (dd *DMMetaData) GetDumpHelper() dbi.DumpHelper {
return new(DumpHelper)
}

25
server/internal/db/dbm/mssql/dialect.go
View File

@@ -239,31 +239,6 @@ func (md *MssqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (md *MssqlDialect) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (md *MssqlDialect) ToColumn(commonColumn *dbi.Column) {
ctype := mssqlColumnTypeMap[commonColumn.DataType]
meta := md.dc.GetMetaData()
if ctype == "" {
commonColumn.DataType = "varchar"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
meta.FixColumn(commonColumn)
}
}
func (md *MssqlDialect) CreateTable(columns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
sqlArr := md.dc.GetMetaData().GenerateTableDDL(columns, tableInfo, dropOldTable)
_, err := md.dc.Exec(strings.Join(sqlArr, ";"))

225
server/internal/db/dbm/mssql/helper.go Normal file
View File

@@ -0,0 +1,225 @@
package mssql
import (
"fmt"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
"strings"
"time"
)
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
// mssql数据类型 对应 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"bigint": dbi.CommonTypeBigint,
"numeric": dbi.CommonTypeNumber,
"bit": dbi.CommonTypeInt,
"smallint": dbi.CommonTypeSmallint,
"decimal": dbi.CommonTypeNumber,
"smallmoney": dbi.CommonTypeNumber,
"int": dbi.CommonTypeInt,
"tinyint": dbi.CommonTypeSmallint, // mssql tinyint不支持负数
"money": dbi.CommonTypeNumber,
"float": dbi.CommonTypeNumber, // 近似数字
"real": dbi.CommonTypeVarchar,
"date": dbi.CommonTypeDate, // 日期和时间
"datetimeoffset": dbi.CommonTypeDatetime,
"datetime2": dbi.CommonTypeDatetime,
"smalldatetime": dbi.CommonTypeDatetime,
"datetime": dbi.CommonTypeDatetime,
"time": dbi.CommonTypeTime,
"char": dbi.CommonTypeChar, // 字符串
"varchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"nchar": dbi.CommonTypeChar,
"nvarchar": dbi.CommonTypeVarchar,
"ntext": dbi.CommonTypeText,
"binary": dbi.CommonTypeBinary,
"varbinary": dbi.CommonTypeBinary,
"cursor": dbi.CommonTypeVarchar, // 其他
"rowversion": dbi.CommonTypeVarchar,
"hierarchyid": dbi.CommonTypeVarchar,
"uniqueidentifier": dbi.CommonTypeVarchar,
"sql_variant": dbi.CommonTypeVarchar,
"xml": dbi.CommonTypeText,
"table": dbi.CommonTypeText,
"geometry": dbi.CommonTypeText, // 空间几何类型
"geography": dbi.CommonTypeText, // 空间地理类型
}
// 公共数据类型 对应 mssql数据类型
mssqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "nvarchar",
dbi.CommonTypeChar: "nchar",
dbi.CommonTypeText: "ntext",
dbi.CommonTypeBlob: "ntext",
dbi.CommonTypeLongblob: "ntext",
dbi.CommonTypeLongtext: "ntext",
dbi.CommonTypeBinary: "varbinary",
dbi.CommonTypeMediumblob: "ntext",
dbi.CommonTypeMediumtext: "ntext",
dbi.CommonTypeVarbinary: "varbinary",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "smallint",
dbi.CommonTypeNumber: "decimal",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime2",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "nvarchar",
dbi.CommonTypeJSON: "nvarchar",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
// 如果dataType是datetime而dbColumnValue是string类型，则需要根据类型格式化
str, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
if dataType == dbi.DataTypeDate && ok {
// 尝试用时间格式解析
res, _ := time.Parse(time.DateOnly, str)
return res.Format(time.DateOnly)
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, str)
return res.Format(time.TimeOnly)
}
return anyx.ToString(dbColumnValue)
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (ch *ColumnHelper) ToColumn(commonColumn *dbi.Column) {
ctype := mssqlColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "varchar"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
ch.FixColumn(commonColumn)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
dataType := strings.ToLower(string(column.DataType))
if collx.ArrayAnyMatches([]string{"date", "time"}, dataType) {
// 如果是datetime，精度取NumScale字段
column.CharMaxLength = column.NumScale
} else if collx.ArrayAnyMatches([]string{"int", "bit", "real", "text", "xml"}, dataType) {
// 不显示长度的类型
column.NumPrecision = 0
column.CharMaxLength = 0
} else if collx.ArrayAnyMatches([]string{"numeric", "decimal", "float"}, dataType) {
// 如果是num，长度取精度和小数位数
column.CharMaxLength = 0
} else if collx.ArrayAnyMatches([]string{"nvarchar", "nchar"}, dataType) {
// 如果是nvarchar，可视长度减半
column.CharMaxLength = column.CharMaxLength / 2
}
}
type DumpHelper struct {
dbi.DefaultDumpHelper
}
func (dh *DumpHelper) BeforeInsertSql(quoteSchema string, tableName string) string {
return fmt.Sprintf("set identity_insert %s.%s on ", quoteSchema, tableName)
}

194
server/internal/db/dbm/mssql/metadata.go
View File

@@ -8,9 +8,7 @@ import (
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
@@ -93,6 +91,7 @@ func (md *MssqlMetaData) GetTables(tableNames ...string) ([]dbi.Table, error) {
// 获取列元信息, 如列名等
func (md *MssqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error) {
meta := md.dc.GetMetaData()
columnHelper := meta.GetColumnHelper()
tableName := strings.Join(collx.ArrayMap[string, string](tableNames, func(val string) string {
return fmt.Sprintf("'%s'", meta.RemoveQuote(val))
}), ",")
@@ -119,32 +118,13 @@ func (md *MssqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
NumScale: cast.ToInt(re["NUM_SCALE"]),
}
md.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
return columns, nil
}
func (md *MssqlMetaData) FixColumn(column *dbi.Column) {
dataType := strings.ToLower(string(column.DataType))
if collx.ArrayAnyMatches([]string{"date", "time"}, dataType) {
// 如果是datetime，精度取NumScale字段
column.CharMaxLength = column.NumScale
} else if collx.ArrayAnyMatches([]string{"int", "bit", "real", "text", "xml"}, dataType) {
// 不显示长度的类型
column.NumPrecision = 0
column.CharMaxLength = 0
} else if collx.ArrayAnyMatches([]string{"numeric", "decimal", "float"}, dataType) {
// 如果是num，长度取精度和小数位数
column.CharMaxLength = 0
} else if collx.ArrayAnyMatches([]string{"nvarchar", "nchar"}, dataType) {
// 如果是nvarchar，可视长度减半
column.CharMaxLength = column.CharMaxLength / 2
}
}
// 获取表主键字段名，不存在主键标识则默认第一个字段
func (md *MssqlMetaData) GetPrimaryKey(tablename string) (string, error) {
columns, err := md.GetColumns(tablename)
@@ -427,172 +407,14 @@ func (md *MssqlMetaData) GetIdentifierQuoteString() string {
return "["
}
func (md *MssqlMetaData) BeforeDumpInsertSql(quoteSchema string, tableName string) string {
return fmt.Sprintf("set identity_insert %s.%s on ", quoteSchema, tableName)
func (md *MssqlMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
func (md *MssqlMetaData) GetDataConverter() dbi.DataConverter {
return converter
func (md *MssqlMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
// mssql数据类型 对应 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"bigint": dbi.CommonTypeBigint,
"numeric": dbi.CommonTypeNumber,
"bit": dbi.CommonTypeInt,
"smallint": dbi.CommonTypeSmallint,
"decimal": dbi.CommonTypeNumber,
"smallmoney": dbi.CommonTypeNumber,
"int": dbi.CommonTypeInt,
"tinyint": dbi.CommonTypeSmallint, // mssql tinyint不支持负数
"money": dbi.CommonTypeNumber,
"float": dbi.CommonTypeNumber, // 近似数字
"real": dbi.CommonTypeVarchar,
"date": dbi.CommonTypeDate, // 日期和时间
"datetimeoffset": dbi.CommonTypeDatetime,
"datetime2": dbi.CommonTypeDatetime,
"smalldatetime": dbi.CommonTypeDatetime,
"datetime": dbi.CommonTypeDatetime,
"time": dbi.CommonTypeTime,
"char": dbi.CommonTypeChar, // 字符串
"varchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"nchar": dbi.CommonTypeChar,
"nvarchar": dbi.CommonTypeVarchar,
"ntext": dbi.CommonTypeText,
"binary": dbi.CommonTypeBinary,
"varbinary": dbi.CommonTypeBinary,
"cursor": dbi.CommonTypeVarchar, // 其他
"rowversion": dbi.CommonTypeVarchar,
"hierarchyid": dbi.CommonTypeVarchar,
"uniqueidentifier": dbi.CommonTypeVarchar,
"sql_variant": dbi.CommonTypeVarchar,
"xml": dbi.CommonTypeText,
"table": dbi.CommonTypeText,
"geometry": dbi.CommonTypeText, // 空间几何类型
"geography": dbi.CommonTypeText, // 空间地理类型
}
// 公共数据类型 对应 mssql数据类型
mssqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "nvarchar",
dbi.CommonTypeChar: "nchar",
dbi.CommonTypeText: "ntext",
dbi.CommonTypeBlob: "ntext",
dbi.CommonTypeLongblob: "ntext",
dbi.CommonTypeLongtext: "ntext",
dbi.CommonTypeBinary: "varbinary",
dbi.CommonTypeMediumblob: "ntext",
dbi.CommonTypeMediumtext: "ntext",
dbi.CommonTypeVarbinary: "varbinary",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "smallint",
dbi.CommonTypeNumber: "decimal",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime2",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "nvarchar",
dbi.CommonTypeJSON: "nvarchar",
}
)
type DataConverter struct {
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
// 如果dataType是datetime而dbColumnValue是string类型，则需要根据类型格式化
str, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
if dataType == dbi.DataTypeDate && ok {
// 尝试用时间格式解析
res, _ := time.Parse(time.DateOnly, str)
return res.Format(time.DateOnly)
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, str)
return res.Format(time.TimeOnly)
}
return anyx.ToString(dbColumnValue)
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (md *MssqlMetaData) GetDumpHelper() dbi.DumpHelper {
return new(DumpHelper)
}

28
server/internal/db/dbm/mysql/dialect.go
View File

@@ -51,10 +51,6 @@ func (md *MysqlDialect) BatchInsert(tx *sql.Tx, tableName string, columns []stri
return md.dc.TxExec(tx, sqlStr, args...)
}
func (md *MysqlDialect) GetDataConverter() dbi.DataConverter {
return converter
}
func (md *MysqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
tableName := copy.TableName
@@ -77,30 +73,6 @@ func (md *MysqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (md *MysqlDialect) ToCommonColumn(column *dbi.Column) {
dataType := column.DataType
t1 := commonColumnTypeMap[string(dataType)]
commonColumnType := dbi.CommonTypeVarchar
if t1 != "" {
commonColumnType = t1
}
column.DataType = commonColumnType
}
func (md *MysqlDialect) ToColumn(column *dbi.Column) {
ctype := mysqlColumnTypeMap[column.DataType]
if ctype == "" {
column.DataType = "varchar"
column.CharMaxLength = 1000
} else {
column.DataType = dbi.ColumnDataType(ctype)
md.dc.GetMetaData().FixColumn(column)
}
}
func (md *MysqlDialect) CreateTable(columns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
sqlArr := md.dc.GetMetaData().GenerateTableDDL(columns, tableInfo, dropOldTable)
for _, sqlStr := range sqlArr {

202
server/internal/db/dbm/mysql/helper.go Normal file
View File

@@ -0,0 +1,202 @@
package mysql
import (
"fmt"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"regexp"
"strings"
"time"
)
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
// mysql数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"bigint": dbi.CommonTypeBigint,
"binary": dbi.CommonTypeBinary,
"blob": dbi.CommonTypeBlob,
"char": dbi.CommonTypeChar,
"datetime": dbi.CommonTypeDatetime,
"date": dbi.CommonTypeDate,
"decimal": dbi.CommonTypeNumber,
"double": dbi.CommonTypeNumber,
"enum": dbi.CommonTypeEnum,
"float": dbi.CommonTypeNumber,
"int": dbi.CommonTypeInt,
"json": dbi.CommonTypeJSON,
"longblob": dbi.CommonTypeLongblob,
"longtext": dbi.CommonTypeLongtext,
"mediumblob": dbi.CommonTypeBlob,
"mediumtext": dbi.CommonTypeText,
"set": dbi.CommonTypeVarchar,
"smallint": dbi.CommonTypeSmallint,
"text": dbi.CommonTypeText,
"time": dbi.CommonTypeTime,
"timestamp": dbi.CommonTypeTimestamp,
"tinyint": dbi.CommonTypeTinyint,
"varbinary": dbi.CommonTypeVarbinary,
"varchar": dbi.CommonTypeVarchar,
}
// 公共数据类型 映射 mysql数据类型
mysqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "varchar",
dbi.CommonTypeChar: "char",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "blob",
dbi.CommonTypeLongblob: "longblob",
dbi.CommonTypeLongtext: "longtext",
dbi.CommonTypeBinary: "binary",
dbi.CommonTypeMediumblob: "blob",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "varbinary",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "tinyint",
dbi.CommonTypeNumber: "decimal",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "enum",
dbi.CommonTypeJSON: "json",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
// 如果dataType是datetime而dbColumnValue是string类型，则需要根据类型格式化
str, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, str)
return res.Format(time.DateOnly)
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, str)
return res.Format(time.TimeOnly)
}
return anyx.ToString(dbColumnValue)
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.DateTime, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
// mysql时间类型无需格式化
return fmt.Sprintf("'%s'", dbColumnValue)
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(dialectColumn *dbi.Column) {
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
commonColumnType := dbi.CommonTypeVarchar
if t1 != "" {
commonColumnType = t1
}
dialectColumn.DataType = commonColumnType
}
func (ch *ColumnHelper) ToColumn(column *dbi.Column) {
ctype := mysqlColumnTypeMap[column.DataType]
if ctype == "" {
column.DataType = "varchar"
column.CharMaxLength = 1000
} else {
column.DataType = dbi.ColumnDataType(ctype)
ch.FixColumn(column)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
// 如果是int整型，删除精度
if strings.Contains(strings.ToLower(string(column.DataType)), "int") {
column.NumScale = 0
column.CharMaxLength = 0
} else
// 如果是text，删除长度
if strings.Contains(strings.ToLower(string(column.DataType)), "text") {
column.CharMaxLength = 0
column.NumPrecision = 0
}
}

179
server/internal/db/dbm/mysql/metadata.go
View File

@@ -6,12 +6,9 @@ import (
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/kanzihuang/vitess/go/vt/sqlparser"
"github.com/may-fly/cast"
@@ -91,6 +88,7 @@ func (md *MysqlMetaData) GetTables(tableNames ...string) ([]dbi.Table, error) {
// 获取列元信息, 如列名等
func (md *MysqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error) {
meta := md.dc.GetMetaData()
columnHelper := meta.GetColumnHelper()
tableName := strings.Join(collx.ArrayMap[string, string](tableNames, func(val string) string {
return fmt.Sprintf("'%s'", meta.RemoveQuote(val))
}), ",")
@@ -117,25 +115,12 @@ func (md *MysqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
NumScale: cast.ToInt(re["numScale"]),
}
md.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
return columns, nil
}
func (md *MysqlMetaData) FixColumn(column *dbi.Column) {
// 如果是int整型，删除精度
if strings.Contains(strings.ToLower(string(column.DataType)), "int") {
column.NumScale = 0
column.CharMaxLength = 0
} else
// 如果是text，删除长度
if strings.Contains(strings.ToLower(string(column.DataType)), "text") {
column.CharMaxLength = 0
column.NumPrecision = 0
}
}
// 获取表主键字段名，不存在主键标识则默认第一个字段
func (md *MysqlMetaData) GetPrimaryKey(tablename string) (string, error) {
columns, err := md.GetColumns(tablename)
@@ -345,164 +330,14 @@ func (md *MysqlMetaData) QuoteLiteral(literal string) string {
return "'" + literal + "'"
}
func (md *MysqlMetaData) SqlParserDialect() sqlparser.Dialect {
func (md *MysqlMetaData) GetSqlParserDialect() sqlparser.Dialect {
return sqlparser.MysqlDialect{}
}
func (md *MysqlMetaData) GetDataConverter() dbi.DataConverter {
return converter
func (md *MysqlMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
converter = new(DataConverter)
// mysql数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"bigint": dbi.CommonTypeBigint,
"binary": dbi.CommonTypeBinary,
"blob": dbi.CommonTypeBlob,
"char": dbi.CommonTypeChar,
"datetime": dbi.CommonTypeDatetime,
"date": dbi.CommonTypeDate,
"decimal": dbi.CommonTypeNumber,
"double": dbi.CommonTypeNumber,
"enum": dbi.CommonTypeEnum,
"float": dbi.CommonTypeNumber,
"int": dbi.CommonTypeInt,
"json": dbi.CommonTypeJSON,
"longblob": dbi.CommonTypeLongblob,
"longtext": dbi.CommonTypeLongtext,
"mediumblob": dbi.CommonTypeBlob,
"mediumtext": dbi.CommonTypeText,
"set": dbi.CommonTypeVarchar,
"smallint": dbi.CommonTypeSmallint,
"text": dbi.CommonTypeText,
"time": dbi.CommonTypeTime,
"timestamp": dbi.CommonTypeTimestamp,
"tinyint": dbi.CommonTypeTinyint,
"varbinary": dbi.CommonTypeVarbinary,
"varchar": dbi.CommonTypeVarchar,
}
// 公共数据类型 映射 mysql数据类型
mysqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "varchar",
dbi.CommonTypeChar: "char",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "blob",
dbi.CommonTypeLongblob: "longblob",
dbi.CommonTypeLongtext: "longtext",
dbi.CommonTypeBinary: "binary",
dbi.CommonTypeMediumblob: "blob",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "varbinary",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "tinyint",
dbi.CommonTypeNumber: "decimal",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "enum",
dbi.CommonTypeJSON: "json",
}
)
type DataConverter struct {
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
// 如果dataType是datetime而dbColumnValue是string类型，则需要根据类型格式化
str, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, str)
return res.Format(time.DateOnly)
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, str)
return res.Format(time.TimeOnly)
}
return anyx.ToString(dbColumnValue)
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if ok {
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.DateTime, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
}
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
// mysql时间类型无需格式化
return fmt.Sprintf("'%s'", dbColumnValue)
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (md *MysqlMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}

27
server/internal/db/dbm/oracle/dialect.go
View File

@@ -155,33 +155,6 @@ func (od *OracleDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (od *OracleDialect) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
// 如果是number类型，需要根据公共类型加上长度, 如 bigint 需要转换为number(19,0)
if strings.Contains(string(t1), "NUMBER") {
dialectColumn.CharMaxLength = 19
}
}
}
func (od *OracleDialect) ToColumn(commonColumn *dbi.Column) {
ctype := oracleColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "NVARCHAR2"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
od.dc.GetMetaData().FixColumn(commonColumn)
}
}
func (od *OracleDialect) CreateTable(commonColumns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
meta := od.dc.GetMetaData()
sqlArr := meta.GenerateTableDDL(commonColumns, tableInfo, dropOldTable)

179
server/internal/db/dbm/oracle/helper.go Normal file
View File

@@ -0,0 +1,179 @@
package oracle
import (
"fmt"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
var (
// 数字类型
numberTypeRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeTypeRegexp = regexp.MustCompile(`(?i)date|timestamp`)
// oracle数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"CHAR": dbi.CommonTypeChar,
"NCHAR": dbi.CommonTypeChar,
"VARCHAR2": dbi.CommonTypeVarchar,
"NVARCHAR2": dbi.CommonTypeVarchar,
"NUMBER": dbi.CommonTypeNumber,
"INTEGER": dbi.CommonTypeInt,
"INT": dbi.CommonTypeInt,
"DECIMAL": dbi.CommonTypeNumber,
"FLOAT": dbi.CommonTypeNumber,
"REAL": dbi.CommonTypeNumber,
"BINARY_FLOAT": dbi.CommonTypeNumber,
"BINARY_DOUBLE": dbi.CommonTypeNumber,
"DATE": dbi.CommonTypeDate,
"TIMESTAMP": dbi.CommonTypeDatetime,
"LONG": dbi.CommonTypeLongtext,
"BLOB": dbi.CommonTypeLongtext,
"CLOB": dbi.CommonTypeLongtext,
"NCLOB": dbi.CommonTypeLongtext,
"BFILE": dbi.CommonTypeBinary,
}
// 公共数据类型 映射 oracle数据类型
oracleColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "NVARCHAR2",
dbi.CommonTypeChar: "NCHAR",
dbi.CommonTypeText: "CLOB",
dbi.CommonTypeBlob: "CLOB",
dbi.CommonTypeLongblob: "CLOB",
dbi.CommonTypeLongtext: "CLOB",
dbi.CommonTypeBinary: "BFILE",
dbi.CommonTypeMediumblob: "CLOB",
dbi.CommonTypeMediumtext: "CLOB",
dbi.CommonTypeVarbinary: "BFILE",
dbi.CommonTypeInt: "INT",
dbi.CommonTypeSmallint: "INT",
dbi.CommonTypeTinyint: "INT",
dbi.CommonTypeNumber: "NUMBER",
dbi.CommonTypeBigint: "NUMBER",
dbi.CommonTypeDatetime: "DATE",
dbi.CommonTypeDate: "DATE",
dbi.CommonTypeTime: "DATE",
dbi.CommonTypeTimestamp: "TIMESTAMP",
dbi.CommonTypeEnum: "CLOB",
dbi.CommonTypeJSON: "CLOB",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberTypeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeTypeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
// oracle把日期类型数据格式化输出
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
return str
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// oracle把日期类型的数据转化为time类型
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.RFC3339, cast.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("to_timestamp('%s', 'yyyy-mm-dd hh24:mi:ss')", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
// 如果是number类型，需要根据公共类型加上长度, 如 bigint 需要转换为number(19,0)
if strings.Contains(string(t1), "NUMBER") {
dialectColumn.CharMaxLength = 19
}
}
}
func (ch *ColumnHelper) ToColumn(commonColumn *dbi.Column) {
ctype := oracleColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "NVARCHAR2"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
ch.FixColumn(commonColumn)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
// 如果默认值包含.nextval，说明是序列，默认值为null
if strings.Contains(column.ColumnDefault, ".nextval") {
column.ColumnDefault = ""
}
// 统一处理一下数据类型的长度
if collx.ArrayAnyMatches([]string{"date", "time", "lob", "int"}, strings.ToLower(string(column.DataType))) {
// 如果是不需要设置长度的类型
column.CharMaxLength = 0
column.NumPrecision = 0
} else if strings.Contains(strings.ToLower(string(column.DataType)), "char") {
// 如果是字符串类型，长度最大4000，否则修改字段类型为clob
if column.CharMaxLength > 4000 {
column.DataType = "NCLOB"
column.CharMaxLength = 0
column.NumPrecision = 0
}
}
}

150
server/internal/db/dbm/oracle/metadata.go
View File

@@ -5,12 +5,9 @@ import (
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
@@ -118,6 +115,7 @@ func (od *OracleMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
return nil, err
}
columnHelper := meta.GetColumnHelper()
columns := make([]dbi.Column, 0)
for _, re := range res {
column := dbi.Column{
@@ -134,33 +132,12 @@ func (od *OracleMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
NumScale: cast.ToInt(re["NUM_SCALE"]),
}
od.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
return columns, nil
}
func (od *OracleMetaData) FixColumn(column *dbi.Column) {
// 如果默认值包含.nextval，说明是序列，默认值为null
if strings.Contains(column.ColumnDefault, ".nextval") {
column.ColumnDefault = ""
}
// 统一处理一下数据类型的长度
if collx.ArrayAnyMatches([]string{"date", "time", "lob", "int"}, strings.ToLower(string(column.DataType))) {
// 如果是不需要设置长度的类型
column.CharMaxLength = 0
column.NumPrecision = 0
} else if strings.Contains(strings.ToLower(string(column.DataType)), "char") {
// 如果是字符串类型，长度最大4000，否则修改字段类型为clob
if column.CharMaxLength > 4000 {
column.DataType = "NCLOB"
column.CharMaxLength = 0
column.NumPrecision = 0
}
}
}
func (od *OracleMetaData) GetPrimaryKey(tablename string) (string, error) {
columns, err := od.GetColumns(tablename)
if err != nil {
@@ -378,125 +355,10 @@ func (od *OracleMetaData) GetSchemas() ([]string, error) {
return schemaNames, nil
}
func (od *OracleMetaData) GetDataConverter() dbi.DataConverter {
return converter
func (od *OracleMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
var (
// 数字类型
numberTypeRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeTypeRegexp = regexp.MustCompile(`(?i)date|timestamp`)
bracketsRegexp = regexp.MustCompile(`\((\d+)\)`)
converter = new(DataConverter)
// oracle数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"CHAR": dbi.CommonTypeChar,
"NCHAR": dbi.CommonTypeChar,
"VARCHAR2": dbi.CommonTypeVarchar,
"NVARCHAR2": dbi.CommonTypeVarchar,
"NUMBER": dbi.CommonTypeNumber,
"INTEGER": dbi.CommonTypeInt,
"INT": dbi.CommonTypeInt,
"DECIMAL": dbi.CommonTypeNumber,
"FLOAT": dbi.CommonTypeNumber,
"REAL": dbi.CommonTypeNumber,
"BINARY_FLOAT": dbi.CommonTypeNumber,
"BINARY_DOUBLE": dbi.CommonTypeNumber,
"DATE": dbi.CommonTypeDate,
"TIMESTAMP": dbi.CommonTypeDatetime,
"LONG": dbi.CommonTypeLongtext,
"BLOB": dbi.CommonTypeLongtext,
"CLOB": dbi.CommonTypeLongtext,
"NCLOB": dbi.CommonTypeLongtext,
"BFILE": dbi.CommonTypeBinary,
}
// 公共数据类型 映射 oracle数据类型
oracleColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "NVARCHAR2",
dbi.CommonTypeChar: "NCHAR",
dbi.CommonTypeText: "CLOB",
dbi.CommonTypeBlob: "CLOB",
dbi.CommonTypeLongblob: "CLOB",
dbi.CommonTypeLongtext: "CLOB",
dbi.CommonTypeBinary: "BFILE",
dbi.CommonTypeMediumblob: "CLOB",
dbi.CommonTypeMediumtext: "CLOB",
dbi.CommonTypeVarbinary: "BFILE",
dbi.CommonTypeInt: "INT",
dbi.CommonTypeSmallint: "INT",
dbi.CommonTypeTinyint: "INT",
dbi.CommonTypeNumber: "NUMBER",
dbi.CommonTypeBigint: "NUMBER",
dbi.CommonTypeDatetime: "DATE",
dbi.CommonTypeDate: "DATE",
dbi.CommonTypeTime: "DATE",
dbi.CommonTypeTimestamp: "TIMESTAMP",
dbi.CommonTypeEnum: "CLOB",
dbi.CommonTypeJSON: "CLOB",
}
)
type DataConverter struct {
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberTypeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeTypeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
// oracle把日期类型数据格式化输出
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
}
return str
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// oracle把日期类型的数据转化为time类型
if dataType == dbi.DataTypeDateTime {
res, _ := time.Parse(time.RFC3339, cast.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("to_timestamp('%s', 'yyyy-mm-dd hh24:mi:ss')", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (od *OracleMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}

26
server/internal/db/dbm/postgres/dialect.go
View File

@@ -145,7 +145,6 @@ func (pd *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
for _, re := range res {
colName := cast.ToString(re["column_name"])
if colName != "" {
// 查询自增列当前最大值
_, maxRes, err := pd.dc.Query(fmt.Sprintf("select max(%s) max_val from %s", colName, tableName))
if err != nil {
@@ -177,31 +176,6 @@ func (pd *PgsqlDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (pd *PgsqlDialect) ToCommonColumn(column *dbi.Column) {
// 翻译为通用数据库类型
dataType := column.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
column.DataType = dbi.CommonTypeVarchar
column.CharMaxLength = 2000
} else {
column.DataType = t1
}
}
func (pd *PgsqlDialect) ToColumn(commonColumn *dbi.Column) {
ctype := pgsqlColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "varchar"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
}
}
func (pd *PgsqlDialect) CreateTable(commonColumns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
meta := pd.dc.GetMetaData()
sqlArr := meta.GenerateTableDDL(commonColumns, tableInfo, dropOldTable)

229
server/internal/db/dbm/postgres/helper.go Normal file
View File

@@ -0,0 +1,229 @@
package postgres
import (
"fmt"
"io"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
// 提取pg默认值， 如：'id'::varchar 提取id ； '-1'::integer 提取-1
defaultValueRegexp = regexp.MustCompile(`'([^']*)'`)
// pgsql数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"int2": dbi.CommonTypeSmallint,
"int4": dbi.CommonTypeInt,
"int8": dbi.CommonTypeBigint,
"numeric": dbi.CommonTypeNumber,
"decimal": dbi.CommonTypeNumber,
"smallserial": dbi.CommonTypeSmallint,
"serial": dbi.CommonTypeInt,
"bigserial": dbi.CommonTypeBigint,
"largeserial": dbi.CommonTypeBigint,
"money": dbi.CommonTypeNumber,
"bool": dbi.CommonTypeTinyint,
"char": dbi.CommonTypeChar,
"character": dbi.CommonTypeChar,
"nchar": dbi.CommonTypeChar,
"varchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"bytea": dbi.CommonTypeText,
"date": dbi.CommonTypeDate,
"time": dbi.CommonTypeTime,
"timestamp": dbi.CommonTypeTimestamp,
}
// 公共数据类型 映射 pgsql数据类型
pgsqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "varchar",
dbi.CommonTypeChar: "char",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "text",
dbi.CommonTypeLongblob: "text",
dbi.CommonTypeLongtext: "text",
dbi.CommonTypeBinary: "text",
dbi.CommonTypeMediumblob: "text",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "text",
dbi.CommonTypeInt: "int4",
dbi.CommonTypeSmallint: "int2",
dbi.CommonTypeTinyint: "int2",
dbi.CommonTypeNumber: "numeric",
dbi.CommonTypeBigint: "int8",
dbi.CommonTypeDatetime: "timestamp",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "varchar(2000)",
dbi.CommonTypeJSON: "varchar(2000)",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := fmt.Sprintf("%v", dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:16:28.545377+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, err = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00Z"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:16:28.545075+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return cast.ToString(dbColumnValue)
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(column *dbi.Column) {
// 翻译为通用数据库类型
dataType := column.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
column.DataType = dbi.CommonTypeVarchar
column.CharMaxLength = 2000
} else {
column.DataType = t1
}
}
func (ch *ColumnHelper) ToColumn(commonColumn *dbi.Column) {
ctype := pgsqlColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "varchar"
commonColumn.CharMaxLength = 2000
} else {
commonColumn.DataType = dbi.ColumnDataType(ctype)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
dataType := strings.ToLower(string(column.DataType))
// 哪些字段可以指定长度
if !collx.ArrayAnyMatches([]string{"char", "time", "bit", "num", "decimal"}, dataType) {
column.CharMaxLength = 0
column.NumPrecision = 0
} else if strings.Contains(dataType, "char") {
// 如果类型是文本，长度翻倍
column.CharMaxLength = column.CharMaxLength * 2
}
// 如果默认值带冒号，如：'id'::varchar
if column.ColumnDefault != "" && strings.Contains(column.ColumnDefault, "::") && !strings.HasPrefix(column.ColumnDefault, "nextval") {
match := defaultValueRegexp.FindStringSubmatch(column.ColumnDefault)
if len(match) > 1 {
column.ColumnDefault = match[1]
}
}
}
type DumpHelper struct {
dbi.DefaultDumpHelper
}
func (dh *DumpHelper) AfterInsert(writer io.Writer, tableName string, columns []dbi.Column) {
// 设置自增序列当前值
for _, column := range columns {
if column.IsIdentity {
seq := fmt.Sprintf("SELECT setval('%s_%s_seq', (SELECT max(%s) FROM %s));\n", tableName, column.ColumnName, column.ColumnName, tableName)
writer.Write([]byte(seq))
}
}
writer.Write([]byte("COMMIT;\n"))
}

184
server/internal/db/dbm/postgres/metadata.go
View File

@@ -6,12 +6,9 @@ import (
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/errorx"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
@@ -100,6 +97,7 @@ func (pd *PgsqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
return nil, err
}
columnHelper := meta.GetColumnHelper()
columns := make([]dbi.Column, 0)
for _, re := range res {
column := dbi.Column{
@@ -115,31 +113,12 @@ func (pd *PgsqlMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
NumPrecision: cast.ToInt(re["numPrecision"]),
NumScale: cast.ToInt(re["numScale"]),
}
pd.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
return columns, nil
}
func (pd *PgsqlMetaData) FixColumn(column *dbi.Column) {
dataType := strings.ToLower(string(column.DataType))
// 哪些字段可以指定长度
if !collx.ArrayAnyMatches([]string{"char", "time", "bit", "num", "decimal"}, dataType) {
column.CharMaxLength = 0
column.NumPrecision = 0
} else if strings.Contains(dataType, "char") {
// 如果类型是文本，长度翻倍
column.CharMaxLength = column.CharMaxLength * 2
}
// 如果默认值带冒号，如：'id'::varchar
if column.ColumnDefault != "" && strings.Contains(column.ColumnDefault, "::") && !strings.HasPrefix(column.ColumnDefault, "nextval") {
match := defaultValueRegexp.FindStringSubmatch(column.ColumnDefault)
if len(match) > 1 {
column.ColumnDefault = match[1]
}
}
}
func (pd *PgsqlMetaData) GetPrimaryKey(tablename string) (string, error) {
columns, err := pd.GetColumns(tablename)
if err != nil {
@@ -429,161 +408,14 @@ func (pd *PgsqlMetaData) AfterDumpInsert(writer io.Writer, tableName string, col
writer.Write([]byte("COMMIT;\n"))
}
func (pd *PgsqlMetaData) GetDataConverter() dbi.DataConverter {
return converter
func (pd *PgsqlMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime|timestamp`)
// 日期类型
dateRegexp = regexp.MustCompile(`(?i)date`)
// 时间类型
timeRegexp = regexp.MustCompile(`(?i)time`)
// 提取pg默认值， 如：'id'::varchar 提取id ； '-1'::integer 提取-1
defaultValueRegexp = regexp.MustCompile(`'([^']*)'`)
converter = new(DataConverter)
// pgsql数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"int2": dbi.CommonTypeSmallint,
"int4": dbi.CommonTypeInt,
"int8": dbi.CommonTypeBigint,
"numeric": dbi.CommonTypeNumber,
"decimal": dbi.CommonTypeNumber,
"smallserial": dbi.CommonTypeSmallint,
"serial": dbi.CommonTypeInt,
"bigserial": dbi.CommonTypeBigint,
"largeserial": dbi.CommonTypeBigint,
"money": dbi.CommonTypeNumber,
"bool": dbi.CommonTypeTinyint,
"char": dbi.CommonTypeChar,
"character": dbi.CommonTypeChar,
"nchar": dbi.CommonTypeChar,
"varchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"bytea": dbi.CommonTypeText,
"date": dbi.CommonTypeDate,
"time": dbi.CommonTypeTime,
"timestamp": dbi.CommonTypeTimestamp,
}
// 公共数据类型 映射 pgsql数据类型
pgsqlColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "varchar",
dbi.CommonTypeChar: "char",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "text",
dbi.CommonTypeLongblob: "text",
dbi.CommonTypeLongtext: "text",
dbi.CommonTypeBinary: "text",
dbi.CommonTypeMediumblob: "text",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "text",
dbi.CommonTypeInt: "int4",
dbi.CommonTypeSmallint: "int2",
dbi.CommonTypeTinyint: "int2",
dbi.CommonTypeNumber: "numeric",
dbi.CommonTypeBigint: "int8",
dbi.CommonTypeDatetime: "timestamp",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "time",
dbi.CommonTypeTimestamp: "timestamp",
dbi.CommonTypeEnum: "varchar(2000)",
dbi.CommonTypeJSON: "varchar(2000)",
}
)
type DataConverter struct {
func (pd *PgsqlMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
// 日期时间类型
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
// 日期类型
if dateRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDate
}
// 时间类型
if timeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := fmt.Sprintf("%v", dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:16:28.545377+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, err = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00Z"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:16:28.545075+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return cast.ToString(dbColumnValue)
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
// 如果dataType是datetime而dbColumnValue是string类型，则需要转换为time.Time类型
_, ok := dbColumnValue.(string)
if dataType == dbi.DataTypeDateTime && ok {
res, _ := time.Parse(time.RFC3339, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeDate && ok {
res, _ := time.Parse(time.DateOnly, anyx.ToString(dbColumnValue))
return res
}
if dataType == dbi.DataTypeTime && ok {
res, _ := time.Parse(time.TimeOnly, anyx.ToString(dbColumnValue))
return res
}
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (pd *PgsqlMetaData) GetDumpHelper() dbi.DumpHelper {
return new(DumpHelper)
}

26
server/internal/db/dbm/sqlite/dialect.go
View File

@@ -50,10 +50,6 @@ func (sd *SqliteDialect) BatchInsert(tx *sql.Tx, tableName string, columns []str
return exec, err
}
func (sd *SqliteDialect) GetDataConverter() dbi.DataConverter {
return converter
}
func (sd *SqliteDialect) CopyTable(copy *dbi.DbCopyTable) error {
tableName := copy.TableName
@@ -86,28 +82,6 @@ func (sd *SqliteDialect) CopyTable(copy *dbi.DbCopyTable) error {
return err
}
func (sd *SqliteDialect) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (sd *SqliteDialect) ToColumn(commonColumn *dbi.Column) {
ctype := sqliteColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "nvarchar"
commonColumn.CharMaxLength = 2000
} else {
sd.dc.GetMetaData().FixColumn(commonColumn)
}
}
func (sd *SqliteDialect) CreateTable(columns []dbi.Column, tableInfo dbi.Table, dropOldTable bool) (int, error) {
sqlArr := sd.dc.GetMetaData().GenerateTableDDL(columns, tableInfo, dropOldTable)
for _, sqlStr := range sqlArr {

184
server/internal/db/dbm/sqlite/helper.go Normal file
View File

@@ -0,0 +1,184 @@
package sqlite
import (
"fmt"
"io"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/utils/anyx"
"regexp"
"strings"
"time"
)
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit|real`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime`)
dataTypeRegexp = regexp.MustCompile(`(\w+)\((\d*),?(\d*)\)`)
dateHelper = new(DataHelper)
// sqlite数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"int": dbi.CommonTypeInt,
"integer": dbi.CommonTypeInt,
"tinyint": dbi.CommonTypeTinyint,
"smallint": dbi.CommonTypeSmallint,
"mediumint": dbi.CommonTypeSmallint,
"bigint": dbi.CommonTypeBigint,
"int2": dbi.CommonTypeInt,
"int8": dbi.CommonTypeInt,
"character": dbi.CommonTypeChar,
"varchar": dbi.CommonTypeVarchar,
"varying character": dbi.CommonTypeVarchar,
"nchar": dbi.CommonTypeChar,
"native character": dbi.CommonTypeVarchar,
"nvarchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"clob": dbi.CommonTypeBlob,
"blob": dbi.CommonTypeBlob,
"real": dbi.CommonTypeNumber,
"double": dbi.CommonTypeNumber,
"double precision": dbi.CommonTypeNumber,
"float": dbi.CommonTypeNumber,
"numeric": dbi.CommonTypeNumber,
"decimal": dbi.CommonTypeNumber,
"boolean": dbi.CommonTypeTinyint,
"date": dbi.CommonTypeDate,
"datetime": dbi.CommonTypeDatetime,
}
// 公共数据类型 映射 sqlite数据类型
sqliteColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "nvarchar",
dbi.CommonTypeChar: "nchar",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "blob",
dbi.CommonTypeLongblob: "blob",
dbi.CommonTypeLongtext: "text",
dbi.CommonTypeBinary: "text",
dbi.CommonTypeMediumblob: "blob",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "text",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "tinyint",
dbi.CommonTypeNumber: "number",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "datetime",
dbi.CommonTypeTimestamp: "datetime",
dbi.CommonTypeEnum: "nvarchar(2000)",
dbi.CommonTypeJSON: "nvarchar(2000)",
}
)
type DataHelper struct {
}
func (dc *DataHelper) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
return dbi.DataTypeString
}
func (dc *DataHelper) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:08:22.275688+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return str
}
func (dc *DataHelper) ParseData(dbColumnValue any, dataType dbi.DataType) any {
return dbColumnValue
}
func (dc *DataHelper) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
}
type ColumnHelper struct {
}
func (ch *ColumnHelper) ToCommonColumn(dialectColumn *dbi.Column) {
// 翻译为通用数据库类型
dataType := dialectColumn.DataType
t1 := commonColumnTypeMap[string(dataType)]
if t1 == "" {
dialectColumn.DataType = dbi.CommonTypeVarchar
dialectColumn.CharMaxLength = 2000
} else {
dialectColumn.DataType = t1
}
}
func (ch *ColumnHelper) ToColumn(commonColumn *dbi.Column) {
ctype := sqliteColumnTypeMap[commonColumn.DataType]
if ctype == "" {
commonColumn.DataType = "nvarchar"
commonColumn.CharMaxLength = 2000
} else {
ch.FixColumn(commonColumn)
}
}
func (ch *ColumnHelper) FixColumn(column *dbi.Column) {
}
type DumpHelper struct {
dbi.DefaultDumpHelper
}
func (db *DumpHelper) BeforeInsert(writer io.Writer, tableName string) {
}
func (db *DumpHelper) AfterInsert(writer io.Writer, tableName string, columns []dbi.Column) {
}

156
server/internal/db/dbm/sqlite/metadata.go
View File

@@ -3,15 +3,12 @@ package sqlite
import (
"errors"
"fmt"
"io"
"mayfly-go/internal/db/dbm/dbi"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/anyx"
"mayfly-go/pkg/utils/collx"
"mayfly-go/pkg/utils/stringx"
"regexp"
"strings"
"time"
"github.com/may-fly/cast"
)
@@ -99,9 +96,8 @@ func (sd *SqliteMetaData) getDataTypes(dataType string) (string, string, string)
// 获取列元信息, 如列名等
func (sd *SqliteMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error) {
columns := make([]dbi.Column, 0)
columnHelper := sd.dc.GetMetaData().GetColumnHelper()
for i := 0; i < len(tableNames); i++ {
tableName := tableNames[i]
_, res, err := sd.dc.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName))
@@ -138,8 +134,7 @@ func (sd *SqliteMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
column.CharMaxLength = cast.ToInt(length)
}
column.DataType = dbi.ColumnDataType(dataType)
sd.FixColumn(&column)
columnHelper.FixColumn(&column)
columns = append(columns, column)
}
@@ -147,9 +142,6 @@ func (sd *SqliteMetaData) GetColumns(tableNames ...string) ([]dbi.Column, error)
return columns, nil
}
func (sd *SqliteMetaData) FixColumn(column *dbi.Column) {
}
func (sd *SqliteMetaData) GetPrimaryKey(tableName string) (string, error) {
_, res, err := sd.dc.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName))
if err != nil {
@@ -318,146 +310,14 @@ func (sd *SqliteMetaData) GetSchemas() ([]string, error) {
return nil, nil
}
func (sd *SqliteMetaData) GetDataConverter() dbi.DataConverter {
return converter
func (sd *SqliteMetaData) GetDataHelper() dbi.DataHelper {
return new(DataHelper)
}
func (sd *SqliteMetaData) BeforeDumpInsert(writer io.Writer, tableName string) {
}
func (sd *SqliteMetaData) AfterDumpInsert(writer io.Writer, tableName string, columns []dbi.Column) {
func (sd *SqliteMetaData) GetColumnHelper() dbi.ColumnHelper {
return new(ColumnHelper)
}
var (
// 数字类型
numberRegexp = regexp.MustCompile(`(?i)int|double|float|number|decimal|byte|bit|real`)
// 日期时间类型
datetimeRegexp = regexp.MustCompile(`(?i)datetime`)
dataTypeRegexp = regexp.MustCompile(`(\w+)\((\d*),?(\d*)\)`)
converter = new(DataConverter)
// sqlite数据类型 映射 公共数据类型
commonColumnTypeMap = map[string]dbi.ColumnDataType{
"int": dbi.CommonTypeInt,
"integer": dbi.CommonTypeInt,
"tinyint": dbi.CommonTypeTinyint,
"smallint": dbi.CommonTypeSmallint,
"mediumint": dbi.CommonTypeSmallint,
"bigint": dbi.CommonTypeBigint,
"int2": dbi.CommonTypeInt,
"int8": dbi.CommonTypeInt,
"character": dbi.CommonTypeChar,
"varchar": dbi.CommonTypeVarchar,
"varying character": dbi.CommonTypeVarchar,
"nchar": dbi.CommonTypeChar,
"native character": dbi.CommonTypeVarchar,
"nvarchar": dbi.CommonTypeVarchar,
"text": dbi.CommonTypeText,
"clob": dbi.CommonTypeBlob,
"blob": dbi.CommonTypeBlob,
"real": dbi.CommonTypeNumber,
"double": dbi.CommonTypeNumber,
"double precision": dbi.CommonTypeNumber,
"float": dbi.CommonTypeNumber,
"numeric": dbi.CommonTypeNumber,
"decimal": dbi.CommonTypeNumber,
"boolean": dbi.CommonTypeTinyint,
"date": dbi.CommonTypeDate,
"datetime": dbi.CommonTypeDatetime,
}
// 公共数据类型 映射 sqlite数据类型
sqliteColumnTypeMap = map[dbi.ColumnDataType]string{
dbi.CommonTypeVarchar: "nvarchar",
dbi.CommonTypeChar: "nchar",
dbi.CommonTypeText: "text",
dbi.CommonTypeBlob: "blob",
dbi.CommonTypeLongblob: "blob",
dbi.CommonTypeLongtext: "text",
dbi.CommonTypeBinary: "text",
dbi.CommonTypeMediumblob: "blob",
dbi.CommonTypeMediumtext: "text",
dbi.CommonTypeVarbinary: "text",
dbi.CommonTypeInt: "int",
dbi.CommonTypeSmallint: "smallint",
dbi.CommonTypeTinyint: "tinyint",
dbi.CommonTypeNumber: "number",
dbi.CommonTypeBigint: "bigint",
dbi.CommonTypeDatetime: "datetime",
dbi.CommonTypeDate: "date",
dbi.CommonTypeTime: "datetime",
dbi.CommonTypeTimestamp: "datetime",
dbi.CommonTypeEnum: "nvarchar(2000)",
dbi.CommonTypeJSON: "nvarchar(2000)",
}
)
type DataConverter struct {
}
func (dc *DataConverter) GetDataType(dbColumnType string) dbi.DataType {
if numberRegexp.MatchString(dbColumnType) {
return dbi.DataTypeNumber
}
if datetimeRegexp.MatchString(dbColumnType) {
return dbi.DataTypeDateTime
}
return dbi.DataTypeString
}
func (dc *DataConverter) FormatData(dbColumnValue any, dataType dbi.DataType) string {
str := anyx.ToString(dbColumnValue)
switch dataType {
case dbi.DataTypeDateTime: // "2024-01-02T22:08:22.275697+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateTime, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateTime)
case dbi.DataTypeDate: // "2024-01-02T00:00:00+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.DateOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.DateOnly)
case dbi.DataTypeTime: // "0000-01-01T22:08:22.275688+08:00"
// 尝试用时间格式解析
res, err := time.Parse(time.TimeOnly, str)
if err == nil {
return str
}
res, _ = time.Parse(time.RFC3339, str)
return res.Format(time.TimeOnly)
}
return str
}
func (dc *DataConverter) ParseData(dbColumnValue any, dataType dbi.DataType) any {
return dbColumnValue
}
func (dc *DataConverter) WrapValue(dbColumnValue any, dataType dbi.DataType) string {
if dbColumnValue == nil {
return "NULL"
}
switch dataType {
case dbi.DataTypeNumber:
return fmt.Sprintf("%v", dbColumnValue)
case dbi.DataTypeString:
val := fmt.Sprintf("%v", dbColumnValue)
// 转义单引号
val = strings.Replace(val, `'`, `''`, -1)
val = strings.Replace(val, `\''`, `\'`, -1)
// 转义换行符
val = strings.Replace(val, "\n", "\\n", -1)
return fmt.Sprintf("'%s'", val)
case dbi.DataTypeDate, dbi.DataTypeDateTime, dbi.DataTypeTime:
return fmt.Sprintf("'%s'", dc.FormatData(dbColumnValue, dataType))
}
return fmt.Sprintf("'%s'", dbColumnValue)
func (sd *SqliteMetaData) GetDumpHelper() dbi.DumpHelper {
return new(DumpHelper)
}