refactor: db元信息获取调整

server/internal/db/api/db.go

@@ -30,7 +30,7 @@ type Db struct {
 	ProjectApp   projectapp.Project
 }
 
-const DEFAULT_COLUMN_SIZE = 500
+const DEFAULT_ROW_SIZE = 1800
 
 // @router /api/dbs [get]
 func (d *Db) Dbs(rc *ctx.ReqCtx) {
@@ -248,8 +248,8 @@ func (d *Db) DumpSql(rc *ctx.ReqCtx) {
 			maCount = countRes[0]["count"].(int)
 		}
 		// 计算需要查询的页数
-		pageNum := maCount / DEFAULT_COLUMN_SIZE
-		if maCount%DEFAULT_COLUMN_SIZE > 0 {
+		pageNum := maCount / DEFAULT_ROW_SIZE
+		if maCount%DEFAULT_ROW_SIZE > 0 {
 			pageNum++
 		}
 
@@ -261,7 +261,7 @@ func (d *Db) DumpSql(rc *ctx.ReqCtx) {
 			sqlTmp = "SELECT * FROM %s OFFSET %d LIMIT %d"
 		}
 		for index := 0; index < pageNum; index++ {
-			sql := fmt.Sprintf(sqlTmp, table, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE)
+			sql := fmt.Sprintf(sqlTmp, table, index*DEFAULT_ROW_SIZE, DEFAULT_ROW_SIZE)
 			columns, result, _ := dbInstance.SelectData(sql)
 
 			insertSql := "INSERT INTO `%s` VALUES (%s);\n"

server/internal/db/application/db.go

@@ -397,6 +397,49 @@ func valueConvert(data []byte, colType *sql.ColumnType) interface{} {
 	return stringV
 }
 
+func innerSelectByDb(db *sql.DB, selectSql string) ([]map[string]interface{}, error) {
+	rows, err := db.Query(selectSql)
+	if err != nil {
+		return nil, err
+	}
+	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
+	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
+	defer func() {
+		if rows != nil {
+			rows.Close()
+		}
+	}()
+	colTypes, _ := rows.ColumnTypes()
+	// 这里表示一行填充数据
+	scans := make([]interface{}, len(colTypes))
+	// 这里表示一行所有列的值，用[]byte表示
+	vals := make([][]byte, len(colTypes))
+	// 这里scans引用vals，把数据填充到[]byte里
+	for k := range vals {
+		scans[k] = &vals[k]
+	}
+
+	result := make([]map[string]interface{}, 0)
+	for rows.Next() {
+		// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
+		err := rows.Scan(scans...)
+		if err != nil {
+			return nil, err
+		}
+		// 每行数据
+		rowData := make(map[string]interface{})
+		// 把vals中的数据复制到row中
+		for i, v := range vals {
+			colType := colTypes[i]
+			colName := colType.Name()
+			rowData[colName] = valueConvert(v, colType)
+		}
+		// 放入结果集
+		result = append(result, rowData)
+	}
+	return result, nil
+}
+
 type PqSqlDialer struct {
 	sshTunnelMachine *machine.SshTunnelMachine
 }
@@ -428,6 +471,12 @@ func (d *DbInstance) SelectData(execSql string) ([]string, []map[string]interfac
 	return SelectDataByDb(d.db, execSql)
 }
 
+// 执行内部查询语句，不返回列名以及不限制行数
+// 依次返回 结果map，错误
+func (d *DbInstance) innerSelect(execSql string) ([]map[string]interface{}, error) {
+	return innerSelectByDb(d.db, execSql)
+}
+
 // 执行 update, insert, delete，建表等sql
 // 返回影响条数和错误
 func (d *DbInstance) Exec(sql string) (int64, error) {

server/internal/db/application/mysql_meta.go

@@ -10,28 +10,24 @@ const (
 	// mysql 表信息元数据
 	MYSQL_TABLE_MA = `SELECT table_name tableName, engine, table_comment tableComment, 
 	create_time createTime from information_schema.tables
-	WHERE table_schema = (SELECT database()) LIMIT 2000`
+	WHERE table_schema = (SELECT database())`
 
 	// mysql 表信息
 	MYSQL_TABLE_INFO = `SELECT table_name tableName, table_comment tableComment, table_rows tableRows,
 	data_length dataLength, index_length indexLength, create_time createTime 
 	FROM information_schema.tables 
-    WHERE table_schema = (SELECT database()) LIMIT 2000`
+    WHERE table_schema = (SELECT database())`
 
 	// mysql 索引信息
 	MYSQL_INDEX_INFO = `SELECT index_name indexName, column_name columnName, index_type indexType,
 	SEQ_IN_INDEX seqInIndex, INDEX_COMMENT indexComment
 	FROM information_schema.STATISTICS 
-    WHERE table_schema = (SELECT database()) AND table_name = '%s' LIMIT 500`
+    WHERE table_schema = (SELECT database()) AND table_name = '%s'`
 
 	// mysql 列信息元数据
 	MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType,
 	column_comment columnComment, column_key columnKey, extra, is_nullable nullable from information_schema.columns
-	WHERE table_name in (%s) AND table_schema = (SELECT database()) ORDER BY tableName, ordinal_position LIMIT %d, %d`
-
-	// mysql 列信息元数据总数
-	MYSQL_COLOUMN_MA_COUNT = `SELECT COUNT(*) maNum from information_schema.columns
-	WHERE table_name in (%s) AND table_schema = (SELECT database())`
+	WHERE table_schema = (SELECT database()) AND table_name in (%s) ORDER BY tableName, ordinal_position`
 )
 
 type MysqlMetadata struct {
@@ -46,42 +42,16 @@ func (mm *MysqlMetadata) GetTables() []map[string]interface{} {
 
 // 获取列元信息, 如列名等
 func (mm *MysqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
-	var sql, tableName string
+	tableName := ""
 	for i := 0; i < len(tableNames); i++ {
 		if i != 0 {
 			tableName = tableName + ", "
 		}
 		tableName = tableName + "'" + tableNames[i] + "'"
 	}
-
-	pageNum := 1
-	// 如果大于一个表，则统计列数并分页获取
-	if len(tableNames) > 1 {
-		countSql := fmt.Sprintf(MYSQL_COLOUMN_MA_COUNT, tableName)
-		_, countRes, _ := mm.di.SelectData(countSql)
-		maCount := 0
-		// 查询出所有列信息总数，手动分页获取所有数据
-		if count64, is64 := countRes[0]["maNum"].(int64); is64 {
-			maCount = int(count64)
-		} else {
-			maCount = countRes[0]["maNum"].(int)
-		}
-
-		// 计算需要查询的页数
-		pageNum = maCount / DEFAULT_COLUMN_SIZE
-		if maCount%DEFAULT_COLUMN_SIZE > 0 {
-			pageNum++
-		}
-	}
-
-	res := make([]map[string]interface{}, 0)
-	for index := 0; index < pageNum; index++ {
-		sql = fmt.Sprintf(MYSQL_COLUMN_MA, tableName, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE)
-		_, result, err := mm.di.SelectData(sql)
-		biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
-		res = append(res, result...)
-	}
-	return res
+	result, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_COLUMN_MA, tableName))
+	biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
+	return result
 }
 
 // 获取表主键字段名，默认第一个字段
@@ -95,18 +65,20 @@ func (mm *MysqlMetadata) GetPrimaryKey(tablename string) string {
 
 // 获取表信息，比GetTableMetedatas获取更详细的表信息
 func (mm *MysqlMetadata) GetTableInfos() []map[string]interface{} {
-	_, res, _ := mm.di.SelectData(MYSQL_TABLE_INFO)
+	res, err := mm.di.innerSelect(MYSQL_TABLE_INFO)
+	biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
 	return res
 }
 
 // 获取表索引信息
 func (mm *MysqlMetadata) GetTableIndex(tableName string) []map[string]interface{} {
-	_, res, _ := mm.di.SelectData(fmt.Sprintf(MYSQL_INDEX_INFO, tableName))
+	res, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_INDEX_INFO, tableName))
+	biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s")
 	return res
 }
 
 // 获取建表ddl
 func (mm *MysqlMetadata) GetCreateTableDdl(tableName string) []map[string]interface{} {
-	_, res, _ := mm.di.SelectData(fmt.Sprintf("show create table %s ", tableName))
+	res, _ := mm.di.innerSelect(fmt.Sprintf("show create table %s ", tableName))
 	return res
 }

server/internal/db/application/pgsql_meta.go

@@ -34,21 +34,7 @@ const (
 		AND C.relname in (%s)
 	ORDER BY
 		C.relname DESC,
-		A.attnum ASC
-	OFFSET %d LIMIT %d	
-	`
-
-	PGSQL_COLUMN_MA_COUNT = `SELECT COUNT(*) "maNum"
-	FROM
-		pg_attribute a LEFT JOIN pg_description d ON d.objoid = a.attrelid 
-		AND d.objsubid = A.attnum
-		LEFT JOIN pg_class c ON A.attrelid = c.oid
-		LEFT JOIN pg_namespace pn ON c.relnamespace = pn.oid
-		LEFT JOIN pg_type t ON a.atttypid = t.oid 
-	WHERE
-		A.attnum >= 0 
-		AND pn.nspname = (select current_schema())
-		AND C.relname in (%s)
+		A.attnum ASC	
 	`
 )
 
@@ -64,41 +50,16 @@ func (pm *PgsqlMetadata) GetTables() []map[string]interface{} {
 
 // 获取列元信息, 如列名等
 func (pm *PgsqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
-	var sql, tableName string
+	tableName := ""
 	for i := 0; i < len(tableNames); i++ {
 		if i != 0 {
 			tableName = tableName + ", "
 		}
 		tableName = tableName + "'" + tableNames[i] + "'"
 	}
-
-	pageNum := 1
-	// 如果大于一个表，则统计列数并分页获取
-	if len(tableNames) > 1 {
-		countSql := fmt.Sprintf(PGSQL_COLUMN_MA_COUNT, tableName)
-		_, countRes, _ := pm.di.SelectData(countSql)
-		maCount := 0
-		// 查询出所有列信息总数，手动分页获取所有数据
-		if count64, is64 := countRes[0]["maNum"].(int64); is64 {
-			maCount = int(count64)
-		} else {
-			maCount = countRes[0]["maNum"].(int)
-		}
-		// 计算需要查询的页数
-		pageNum = maCount / DEFAULT_COLUMN_SIZE
-		if maCount%DEFAULT_COLUMN_SIZE > 0 {
-			pageNum++
-		}
-	}
-
-	res := make([]map[string]interface{}, 0)
-	for index := 0; index < pageNum; index++ {
-		sql = fmt.Sprintf(PGSQL_COLUMN_MA, tableName, index*DEFAULT_COLUMN_SIZE, DEFAULT_COLUMN_SIZE)
-		_, result, err := pm.di.SelectData(sql)
-		biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
-		res = append(res, result...)
-	}
-	return res
+	result, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_COLUMN_MA, tableName))
+	biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
+	return result
 }
 
 // 获取表主键字段名，默认第一个字段