refactor: sqlexec组件重构优化、新增数据库相关系统参数配置、相关问题修复

server/internal/db/api/db.go

@@ -415,6 +415,7 @@ func (d *Db) DeleteSql(rc *req.Ctx) {
 	dbSql := &entity.DbSql{Type: 1, DbId: GetDbId(rc.GinCtx)}
 	dbSql.CreatorId = rc.LoginAccount.Id
 	dbSql.Name = rc.GinCtx.Query("name")
+	dbSql.Db = rc.GinCtx.Query("db")
 
 	model.DeleteByCondition(dbSql)
 

server/internal/db/api/form/db.go

@@ -19,7 +19,7 @@ type DbForm struct {
 }
 
 type DbSqlSaveForm struct {
-	Name string
+	Name string `binding:"required"`
 	Sql  string `binding:"required"`
 	Type int    `binding:"required"`
 	Db   string `binding:"required"`

server/internal/db/application/application.go

@@ -1,6 +1,8 @@
 package application
 
-import "mayfly-go/internal/db/infrastructure/persistence"
+import (
+	"mayfly-go/internal/db/infrastructure/persistence"
+)
 
 var (
 	dbApp        Db        = newDbApp(persistence.GetDbRepo(), persistence.GetDbSqlRepo())

server/internal/db/application/db.go

@@ -159,7 +159,7 @@ func (d *dbAppImpl) GetDatabases(ed *entity.Db) []string {
 	biz.ErrIsNilAppendErr(err, "数据库连接失败: %s")
 	defer dbConn.Close()
 
-	_, res, err := SelectDataByDb(dbConn, getDatabasesSql, true)
+	_, res, err := SelectDataByDb(dbConn, getDatabasesSql)
 	biz.ErrIsNilAppendErr(err, "获取数据库列表失败")
 	for _, re := range res {
 		databases = append(databases, re["dbname"].(string))
@@ -248,7 +248,7 @@ type DbInstance struct {
 // 执行查询语句
 // 依次返回 列名数组，结果map，错误
 func (d *DbInstance) SelectData(execSql string) ([]string, []map[string]interface{}, error) {
-	return SelectDataByDb(d.db, execSql, false)
+	return SelectDataByDb(d.db, execSql)
 }
 
 // 将查询结果映射至struct，可具体参考sqlx库
@@ -259,7 +259,7 @@ func (d *DbInstance) SelectData2Struct(execSql string, dest interface{}) error {
 // 执行内部查询语句，不返回列名以及不限制行数
 // 依次返回 结果map，错误
 func (d *DbInstance) innerSelect(execSql string) ([]map[string]interface{}, error) {
-	_, res, err := SelectDataByDb(d.db, execSql, true)
+	_, res, err := SelectDataByDb(d.db, execSql)
 	return res, err
 }
 
@@ -361,7 +361,7 @@ func GetDbConn(d *entity.Db, db string) (*sql.DB, error) {
 	return DB, nil
 }
 
-func SelectDataByDb(db *sql.DB, selectSql string, isInner bool) ([]string, []map[string]interface{}, error) {
+func SelectDataByDb(db *sql.DB, selectSql string) ([]string, []map[string]interface{}, error) {
 	rows, err := db.Query(selectSql)
 	if err != nil {
 		return nil, nil, err
@@ -388,14 +388,7 @@ func SelectDataByDb(db *sql.DB, selectSql string, isInner bool) ([]string, []map
 	colNames := make([]string, 0)
 	// 是否第一次遍历，列名数组只需第一次遍历时加入
 	isFirst := true
-	rowNum := 0
 	for rows.Next() {
-		rowNum++
-		// 非内部sql，则校验返回结果数量
-		if !isInner {
-			biz.IsTrue(rowNum <= Max_Rows, "结果集 > 2000, 请完善条件或分页信息")
-		}
-
 		// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
 		err := rows.Scan(scans...)
 		if err != nil {

server/internal/db/application/db_sql_exec.go

@@ -5,7 +5,11 @@ import (
 	"fmt"
 	"mayfly-go/internal/db/domain/entity"
 	"mayfly-go/internal/db/domain/repository"
+	sysapp "mayfly-go/internal/sys/application"
+	sysentity "mayfly-go/internal/sys/domain/entity"
+	"mayfly-go/pkg/biz"
 	"mayfly-go/pkg/model"
+	"strconv"
 	"strings"
 
 	"github.com/xwb1989/sqlparser"
@@ -73,32 +77,42 @@ func createSqlExecRecord(execSqlReq *DbSqlExecReq) *entity.DbSqlExec {
 
 func (d *dbSqlExecAppImpl) Exec(execSqlReq *DbSqlExecReq) (*DbSqlExecRes, error) {
 	sql := execSqlReq.Sql
-	stmt, err := sqlparser.Parse(sql)
-	if err != nil {
-		// 就算解析失败也执行sql，让数据库来判断错误
-		//global.Log.Error("sql解析失败: ", err)
-		if strings.HasPrefix(strings.ToLower(execSqlReq.Sql), "select") ||
-			strings.HasPrefix(strings.ToLower(execSqlReq.Sql), "show") {
-			return doSelect(execSqlReq)
-		}
-		// 保存执行记录
-		d.dbSqlExecRepo.Insert(createSqlExecRecord(execSqlReq))
-		return doExec(execSqlReq.Sql, execSqlReq.DbInstance)
-	}
-
 	dbSqlExecRecord := createSqlExecRecord(execSqlReq)
 	var execRes *DbSqlExecRes
 	isSelect := false
+
+	stmt, err := sqlparser.Parse(sql)
+	if err != nil {
+		// 就算解析失败也执行sql，让数据库来判断错误。如果是查询sql则简单判断是否有limit分页参数信息（兼容pgsql）
+		// global.Log.Warnf("sqlparse解析sql[%s]失败: %s", sql, err.Error())
+		lowerSql := strings.ToLower(execSqlReq.Sql)
+		isSelect := strings.HasPrefix(lowerSql, "select")
+		if isSelect {
+			biz.IsTrue(strings.Contains(lowerSql, "limit"), "请完善分页信息")
+		}
+		var execErr error
+		if isSelect || strings.HasPrefix(lowerSql, "show") {
+			execRes, execErr = doRead(execSqlReq)
+		} else {
+			execRes, execErr = doExec(execSqlReq.Sql, execSqlReq.DbInstance)
+		}
+		if execErr != nil {
+			return nil, execErr
+		}
+		d.saveSqlExecLog(isSelect, dbSqlExecRecord)
+		return execRes, nil
+	}
+
 	switch stmt := stmt.(type) {
 	case *sqlparser.Select:
 		isSelect = true
-		execRes, err = doSelect(execSqlReq)
+		execRes, err = doSelect(stmt, execSqlReq)
 	case *sqlparser.Show:
 		isSelect = true
-		execRes, err = doSelect(execSqlReq)
+		execRes, err = doRead(execSqlReq)
 	case *sqlparser.OtherRead:
 		isSelect = true
-		execRes, err = doSelect(execSqlReq)
+		execRes, err = doRead(execSqlReq)
 	case *sqlparser.Update:
 		execRes, err = doUpdate(stmt, execSqlReq, dbSqlExecRecord)
 	case *sqlparser.Delete:
@@ -111,12 +125,22 @@ func (d *dbSqlExecAppImpl) Exec(execSqlReq *DbSqlExecReq) (*DbSqlExecRes, error)
 	if err != nil {
 		return nil, err
 	}
+	d.saveSqlExecLog(isSelect, dbSqlExecRecord)
+	return execRes, nil
+}
 
-	if !isSelect {
-		// 保存执行记录
+// 保存sql执行记录，如果是查询类则根据系统配置判断是否保存
+func (d *dbSqlExecAppImpl) saveSqlExecLog(isQuery bool, dbSqlExecRecord *entity.DbSqlExec) {
+	if !isQuery {
+		d.dbSqlExecRepo.Insert(dbSqlExecRecord)
+		return
+	}
+	if sysapp.GetConfigApp().GetConfig(sysentity.ConfigKeyDbSaveQuerySQL).BoolValue(false) {
+		dbSqlExecRecord.Table = "-"
+		dbSqlExecRecord.OldValue = "-"
+		dbSqlExecRecord.Type = entity.DbSqlExecTypeQuery
 		d.dbSqlExecRepo.Insert(dbSqlExecRecord)
 	}
-	return execRes, nil
 }
 
 func (d *dbSqlExecAppImpl) DeleteBy(condition *entity.DbSqlExec) {
@@ -127,7 +151,22 @@ func (d *dbSqlExecAppImpl) GetPageList(condition *entity.DbSqlExec, pageParam *m
 	return d.dbSqlExecRepo.GetPageList(condition, pageParam, toEntity, orderBy...)
 }
 
-func doSelect(execSqlReq *DbSqlExecReq) (*DbSqlExecRes, error) {
+func doSelect(selectStmt *sqlparser.Select, execSqlReq *DbSqlExecReq) (*DbSqlExecRes, error) {
+	selectExprsStr := sqlparser.String(selectStmt.SelectExprs)
+	if selectExprsStr == "*" || strings.Contains(selectExprsStr, ".*") ||
+		len(strings.Split(selectExprsStr, ",")) > 1 {
+		limit := selectStmt.Limit
+		biz.NotNil(limit, "请完善分页信息后执行")
+		count, err := strconv.Atoi(sqlparser.String(limit.Rowcount))
+		biz.ErrIsNil(err, "分页参数有误")
+		maxCount := sysapp.GetConfigApp().GetConfig(sysentity.ConfigKeyDbQueryMaxCount).IntValue(200)
+		biz.IsTrue(count <= maxCount, fmt.Sprintf("查询结果集数需小于系统配置的%d条", maxCount))
+	}
+
+	return doRead(execSqlReq)
+}
+
+func doRead(execSqlReq *DbSqlExecReq) (*DbSqlExecRes, error) {
 	dbInstance := execSqlReq.DbInstance
 	sql := execSqlReq.Sql
 	colNames, res, err := dbInstance.SelectData(sql)

server/internal/db/domain/entity/db_sql_exec.go

@@ -19,4 +19,5 @@ const (
 	DbSqlExecTypeUpdate int8 = 1 // 更新类型
 	DbSqlExecTypeDelete int8 = 2 // 删除类型
 	DbSqlExecTypeInsert int8 = 3 // 插入类型
+	DbSqlExecTypeQuery  int8 = 4 // 查询类型，如select、show等
 )

server/internal/sys/application/config_app.go

@@ -1,12 +1,17 @@
 package application
 
 import (
+	"encoding/json"
 	"mayfly-go/internal/sys/domain/entity"
 	"mayfly-go/internal/sys/domain/repository"
+	"mayfly-go/pkg/cache"
 	"mayfly-go/pkg/global"
 	"mayfly-go/pkg/model"
+	"mayfly-go/pkg/utils"
 )
 
+const SysConfigKeyPrefix = "sys:config:"
+
 type Config interface {
 	GetPageList(condition *entity.Config, pageParam *model.PageParam, toEntity interface{}, orderBy ...string) *model.PageResult
 
@@ -36,12 +41,22 @@ func (a *configAppImpl) Save(config *entity.Config) {
 	} else {
 		a.configRepo.Update(config)
 	}
+	cache.Del(SysConfigKeyPrefix + config.Key)
 }
 
 func (a *configAppImpl) GetConfig(key string) *entity.Config {
 	config := &entity.Config{Key: key}
+	// 优先从缓存中获取
+	cacheStr := cache.GetStr(SysConfigKeyPrefix + key)
+	if cacheStr != "" {
+		json.Unmarshal([]byte(cacheStr), &config)
+		return config
+	}
+
 	if err := a.configRepo.GetConfig(config, "Id", "Key", "Value"); err != nil {
 		global.Log.Warnf("不存在key = [%s] 的系统配置", key)
+	} else {
+		cache.SetStr(SysConfigKeyPrefix+key, utils.ToJsonStr(config))
 	}
 	return config
 }

server/internal/sys/domain/entity/config.go

@@ -3,10 +3,13 @@ package entity
 import (
 	"encoding/json"
 	"mayfly-go/pkg/model"
+	"strconv"
 )
 
 const (
 	ConfigKeyUseLoginCaptcha string = "UseLoginCaptcha" // 是否使用登录验证码
+	ConfigKeyDbQueryMaxCount string = "DbQueryMaxCount" // 数据库查询的最大数量
+	ConfigKeyDbSaveQuerySQL  string = "DbSaveQuerySQL"  // 数据库是否记录查询相关sql
 )
 
 type Config struct {
@@ -41,3 +44,16 @@ func (c *Config) GetJsonMap() map[string]string {
 	_ = json.Unmarshal([]byte(c.Value), &res)
 	return res
 }
+
+// 获取配置的int值，如果配置值非int或不存在，则返回默认值
+func (c *Config) IntValue(defaultValue int) int {
+	// 如果值不存在，则返回默认值
+	if c.Id == 0 {
+		return defaultValue
+	}
+	if intV, err := strconv.Atoi(c.Value); err != nil {
+		return defaultValue
+	} else {
+		return intV
+	}
+}