From 575947795a49b351da04d78c708a5e2fad8d957c Mon Sep 17 00:00:00 2001 From: "meilin.huang" <954537473@qq.com> Date: Sun, 16 Oct 2022 11:01:45 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20db=E5=85=83=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/internal/db/api/db.go | 8 +-- server/internal/db/application/db.go | 49 ++++++++++++++++++ server/internal/db/application/mysql_meta.go | 54 +++++--------------- server/internal/db/application/pgsql_meta.go | 49 ++---------------- 4 files changed, 71 insertions(+), 89 deletions(-) diff --git a/server/internal/db/api/db.go b/server/internal/db/api/db.go index 97e41ea6..7a42cca3 100644 --- a/server/internal/db/api/db.go +++ b/server/internal/db/api/db.go @@ -30,7 +30,7 @@ type Db struct { ProjectApp projectapp.Project } -const DEFAULT_COLUMN_SIZE = 500 +const DEFAULT_ROW_SIZE = 1800 // @router /api/dbs [get] func (d *Db) Dbs(rc *ctx.ReqCtx) { @@ -248,8 +248,8 @@ func (d *Db) DumpSql(rc *ctx.ReqCtx) { maCount = countRes[0]["count"].(int) } // 计算需要查询的页数 - pageNum := maCount / DEFAULT_COLUMN_SIZE - if maCount%DEFAULT_COLUMN_SIZE > 0 { + pageNum := maCount / DEFAULT_ROW_SIZE + if maCount%DEFAULT_ROW_SIZE > 0 { pageNum++ } @@ -261,7 +261,7 @@ func (d *Db) DumpSql(rc *ctx.ReqCtx) { sqlTmp = "SELECT * FROM %s OFFSET %d LIMIT %d" } for index := 0; index < pageNum; index++ { - sql := fmt.Sprintf(sqlTmp, table, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE) + sql := fmt.Sprintf(sqlTmp, table, index*DEFAULT_ROW_SIZE, DEFAULT_ROW_SIZE) columns, result, _ := dbInstance.SelectData(sql) insertSql := "INSERT INTO `%s` VALUES (%s);\n" diff --git a/server/internal/db/application/db.go b/server/internal/db/application/db.go index d6f1625d..a4438132 100644 --- a/server/internal/db/application/db.go +++ b/server/internal/db/application/db.go @@ -397,6 +397,49 @@ func valueConvert(data []byte, colType *sql.ColumnType) interface{} { return stringV } +func innerSelectByDb(db *sql.DB, selectSql string) ([]map[string]interface{}, error) { + rows, err := db.Query(selectSql) + if err != nil { + return nil, err + } + // rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数， + // 后面的链接过来直接报错或拒绝，实际上也没有起效果 + defer func() { + if rows != nil { + rows.Close() + } + }() + colTypes, _ := rows.ColumnTypes() + // 这里表示一行填充数据 + scans := make([]interface{}, len(colTypes)) + // 这里表示一行所有列的值，用[]byte表示 + vals := make([][]byte, len(colTypes)) + // 这里scans引用vals，把数据填充到[]byte里 + for k := range vals { + scans[k] = &vals[k] + } + + result := make([]map[string]interface{}, 0) + for rows.Next() { + // 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大 + err := rows.Scan(scans...) + if err != nil { + return nil, err + } + // 每行数据 + rowData := make(map[string]interface{}) + // 把vals中的数据复制到row中 + for i, v := range vals { + colType := colTypes[i] + colName := colType.Name() + rowData[colName] = valueConvert(v, colType) + } + // 放入结果集 + result = append(result, rowData) + } + return result, nil +} + type PqSqlDialer struct { sshTunnelMachine *machine.SshTunnelMachine } @@ -428,6 +471,12 @@ func (d *DbInstance) SelectData(execSql string) ([]string, []map[string]interfac return SelectDataByDb(d.db, execSql) } +// 执行内部查询语句，不返回列名以及不限制行数 +// 依次返回 结果map，错误 +func (d *DbInstance) innerSelect(execSql string) ([]map[string]interface{}, error) { + return innerSelectByDb(d.db, execSql) +} + // 执行 update, insert, delete，建表等sql // 返回影响条数和错误 func (d *DbInstance) Exec(sql string) (int64, error) { diff --git a/server/internal/db/application/mysql_meta.go b/server/internal/db/application/mysql_meta.go index d5555fb7..e6398809 100644 --- a/server/internal/db/application/mysql_meta.go +++ b/server/internal/db/application/mysql_meta.go @@ -10,28 +10,24 @@ const ( // mysql 表信息元数据 MYSQL_TABLE_MA = `SELECT table_name tableName, engine, table_comment tableComment, create_time createTime from information_schema.tables - WHERE table_schema = (SELECT database()) LIMIT 2000` + WHERE table_schema = (SELECT database())` // mysql 表信息 MYSQL_TABLE_INFO = `SELECT table_name tableName, table_comment tableComment, table_rows tableRows, data_length dataLength, index_length indexLength, create_time createTime FROM information_schema.tables - WHERE table_schema = (SELECT database()) LIMIT 2000` + WHERE table_schema = (SELECT database())` // mysql 索引信息 MYSQL_INDEX_INFO = `SELECT index_name indexName, column_name columnName, index_type indexType, SEQ_IN_INDEX seqInIndex, INDEX_COMMENT indexComment FROM information_schema.STATISTICS - WHERE table_schema = (SELECT database()) AND table_name = '%s' LIMIT 500` + WHERE table_schema = (SELECT database()) AND table_name = '%s'` // mysql 列信息元数据 MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType, column_comment columnComment, column_key columnKey, extra, is_nullable nullable from information_schema.columns - WHERE table_name in (%s) AND table_schema = (SELECT database()) ORDER BY tableName, ordinal_position LIMIT %d, %d` - - // mysql 列信息元数据总数 - MYSQL_COLOUMN_MA_COUNT = `SELECT COUNT(*) maNum from information_schema.columns - WHERE table_name in (%s) AND table_schema = (SELECT database())` + WHERE table_schema = (SELECT database()) AND table_name in (%s) ORDER BY tableName, ordinal_position` ) type MysqlMetadata struct { @@ -46,42 +42,16 @@ func (mm *MysqlMetadata) GetTables() []map[string]interface{} { // 获取列元信息, 如列名等 func (mm *MysqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} { - var sql, tableName string + tableName := "" for i := 0; i < len(tableNames); i++ { if i != 0 { tableName = tableName + ", " } tableName = tableName + "'" + tableNames[i] + "'" } - - pageNum := 1 - // 如果大于一个表，则统计列数并分页获取 - if len(tableNames) > 1 { - countSql := fmt.Sprintf(MYSQL_COLOUMN_MA_COUNT, tableName) - _, countRes, _ := mm.di.SelectData(countSql) - maCount := 0 - // 查询出所有列信息总数，手动分页获取所有数据 - if count64, is64 := countRes[0]["maNum"].(int64); is64 { - maCount = int(count64) - } else { - maCount = countRes[0]["maNum"].(int) - } - - // 计算需要查询的页数 - pageNum = maCount / DEFAULT_COLUMN_SIZE - if maCount%DEFAULT_COLUMN_SIZE > 0 { - pageNum++ - } - } - - res := make([]map[string]interface{}, 0) - for index := 0; index < pageNum; index++ { - sql = fmt.Sprintf(MYSQL_COLUMN_MA, tableName, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE) - _, result, err := mm.di.SelectData(sql) - biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") - res = append(res, result...) - } - return res + result, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_COLUMN_MA, tableName)) + biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") + return result } // 获取表主键字段名，默认第一个字段 @@ -95,18 +65,20 @@ func (mm *MysqlMetadata) GetPrimaryKey(tablename string) string { // 获取表信息，比GetTableMetedatas获取更详细的表信息 func (mm *MysqlMetadata) GetTableInfos() []map[string]interface{} { - _, res, _ := mm.di.SelectData(MYSQL_TABLE_INFO) + res, err := mm.di.innerSelect(MYSQL_TABLE_INFO) + biz.ErrIsNilAppendErr(err, "获取表信息失败: %s") return res } // 获取表索引信息 func (mm *MysqlMetadata) GetTableIndex(tableName string) []map[string]interface{} { - _, res, _ := mm.di.SelectData(fmt.Sprintf(MYSQL_INDEX_INFO, tableName)) + res, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_INDEX_INFO, tableName)) + biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s") return res } // 获取建表ddl func (mm *MysqlMetadata) GetCreateTableDdl(tableName string) []map[string]interface{} { - _, res, _ := mm.di.SelectData(fmt.Sprintf("show create table %s ", tableName)) + res, _ := mm.di.innerSelect(fmt.Sprintf("show create table %s ", tableName)) return res } diff --git a/server/internal/db/application/pgsql_meta.go b/server/internal/db/application/pgsql_meta.go index 450ed2a8..e384ead2 100644 --- a/server/internal/db/application/pgsql_meta.go +++ b/server/internal/db/application/pgsql_meta.go @@ -34,21 +34,7 @@ const ( AND C.relname in (%s) ORDER BY C.relname DESC, - A.attnum ASC - OFFSET %d LIMIT %d - ` - - PGSQL_COLUMN_MA_COUNT = `SELECT COUNT(*) "maNum" - FROM - pg_attribute a LEFT JOIN pg_description d ON d.objoid = a.attrelid - AND d.objsubid = A.attnum - LEFT JOIN pg_class c ON A.attrelid = c.oid - LEFT JOIN pg_namespace pn ON c.relnamespace = pn.oid - LEFT JOIN pg_type t ON a.atttypid = t.oid - WHERE - A.attnum >= 0 - AND pn.nspname = (select current_schema()) - AND C.relname in (%s) + A.attnum ASC ` ) @@ -64,41 +50,16 @@ func (pm *PgsqlMetadata) GetTables() []map[string]interface{} { // 获取列元信息, 如列名等 func (pm *PgsqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} { - var sql, tableName string + tableName := "" for i := 0; i < len(tableNames); i++ { if i != 0 { tableName = tableName + ", " } tableName = tableName + "'" + tableNames[i] + "'" } - - pageNum := 1 - // 如果大于一个表，则统计列数并分页获取 - if len(tableNames) > 1 { - countSql := fmt.Sprintf(PGSQL_COLUMN_MA_COUNT, tableName) - _, countRes, _ := pm.di.SelectData(countSql) - maCount := 0 - // 查询出所有列信息总数，手动分页获取所有数据 - if count64, is64 := countRes[0]["maNum"].(int64); is64 { - maCount = int(count64) - } else { - maCount = countRes[0]["maNum"].(int) - } - // 计算需要查询的页数 - pageNum = maCount / DEFAULT_COLUMN_SIZE - if maCount%DEFAULT_COLUMN_SIZE > 0 { - pageNum++ - } - } - - res := make([]map[string]interface{}, 0) - for index := 0; index < pageNum; index++ { - sql = fmt.Sprintf(PGSQL_COLUMN_MA, tableName, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE) - _, result, err := pm.di.SelectData(sql) - biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") - res = append(res, result...) - } - return res + result, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_COLUMN_MA, tableName)) + biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") + return result } // 获取表主键字段名，默认第一个字段