TRKJ/mayfly-go
2
0
Fork 0
mirror of https://gitee.com/dromara/mayfly-go synced 2025-11-02 15:30:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: pgsql查询列信息解析问题修复等

Browse Source
This commit is contained in:
meilin.huang
2024-10-28 12:13:41 +08:00
parent 5e5afd49dc
commit c1d09f447d
11 changed files with 220 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
build_release.sh
View File

@@ -106,7 +106,7 @@ function buildDocker() {
echo_yellow "-------------------构建docker镜像开始-------------------"
imageVersion=$1
imageName="mayfly/mayfly-go:${imageVersion}"
docker build --platform linux/amd64 --build-arg MAYFLY_GO_VERSION="${imageVersion}" -t "${imageName}" .
docker build --no-cache --platform linux/amd64 --build-arg MAYFLY_GO_VERSION="${imageVersion}" -t "${imageName}" .
echo_green "docker镜像构建完成->[${imageName}]"
echo_yellow "-------------------构建docker镜像结束-------------------"
}
@@ -115,7 +115,7 @@ function buildxDocker() {
echo_yellow "-------------------docker buildx构建镜像开始-------------------"
imageVersion=$1
imageName="ccr.ccs.tencentyun.com/mayfly/mayfly-go:${imageVersion}"
docker buildx build --push --platform linux/amd64,linux/arm64 --build-arg MAYFLY_GO_VERSION="${imageVersion}" -t "${imageName}" .
docker buildx build --no-cache --push --platform linux/amd64,linux/arm64 --build-arg MAYFLY_GO_VERSION="${imageVersion}" -t "${imageName}" .
echo_green "docker多版本镜像构建完成->[${imageName}]"
echo_yellow "-------------------docker buildx构建镜像结束-------------------"
}

2
mayfly_go_web/src/views/flow/flowbiz/dbms/DbSqlExecFlowBizForm.vue
View File

@@ -64,7 +64,7 @@ watch(
);
const changeResourceCode = async (db: any) => {
emit('changeResourceCode', TagResourceTypeEnum.Db.value, db.code);
emit('changeResourceCode', TagResourceTypeEnum.DbName.value, db.code);
};
const validateBizForm = async () => {

118
server/internal/common/utils/stmt.go Normal file
View File

@@ -0,0 +1,118 @@
package utils
import (
"bufio"
"bytes"
"io"
"strings"
"unicode/utf8"
)
// StmtCallback stmt回调函数
type StmtCallback func(stmt string) error
// SplitStmts 语句切割（用于以;结尾为一条语句，并且去除// -- /**/等注释）主要由阿里通义灵码提供
func SplitStmts(r io.Reader, callback StmtCallback) error {
reader := bufio.NewReaderSize(r, 512*1024)
buffer := new(bytes.Buffer) // 使用 bytes.Buffer 来处理数据
var currentStatement bytes.Buffer
var inString bool
var inMultiLineComment bool
var inSingleLineComment bool
var stringDelimiter rune
var escapeNextChar bool // 用于处理转义符
for {
// 读取数据到缓冲区
data, err := reader.ReadBytes('\n') // 按行读取
if err == io.EOF && len(data) == 0 {
break
}
if err != nil && err != io.EOF {
return err
}
buffer.Write(data)
// 处理缓冲区中的数据
for buffer.Len() > 0 {
r, size := utf8.DecodeRune(buffer.Bytes())
if r == utf8.RuneError && size == 1 {
// 如果解码出错，说明数据不完整，继续读取更多数据
break
}
switch {
case inMultiLineComment:
if r == '*' && buffer.Len() >= 2 && buffer.Bytes()[1] == '/' {
inMultiLineComment = false
buffer.Next(2) // 跳过 '*/'
} else {
buffer.Next(size)
}
case inSingleLineComment:
if r == '\n' {
inSingleLineComment = false
}
buffer.Next(size)
case inString:
if escapeNextChar {
// 当前字符是转义后的字符，直接写入。如后一个为" 避免进入r==stringDelimiter判断被当做字符串结束符中断
currentStatement.WriteRune(r)
escapeNextChar = false
} else if r == '\\' {
// 当前字符是转义符，设置标志位并写入
escapeNextChar = true
currentStatement.WriteRune(r)
} else if r == stringDelimiter {
// 当前字符是字符串结束符，结束字符串处理
inString = false
currentStatement.WriteRune(r)
} else {
// 其他字符，直接写入
currentStatement.WriteRune(r)
}
buffer.Next(size)
case r == '/' && buffer.Len() >= 2 && buffer.Bytes()[1] == '*':
inMultiLineComment = true
buffer.Next(2) // 跳过 '/*'
case r == '-' && buffer.Len() >= 2 && buffer.Bytes()[1] == '-':
inSingleLineComment = true
buffer.Next(2) // 跳过 '--'
case r == '\'' || r == '"':
inString = true
stringDelimiter = r
currentStatement.WriteRune(r)
buffer.Next(size)
case r == ';' && !inString && !inMultiLineComment && !inSingleLineComment:
sql := strings.TrimSpace(currentStatement.String())
if sql != "" {
if err := callback(sql); err != nil {
return err
}
}
currentStatement.Reset()
buffer.Next(size)
default:
currentStatement.WriteRune(r)
buffer.Next(size)
}
}
// 如果读取到 EOF 并且缓冲区为空，退出循环
if err == io.EOF && buffer.Len() == 0 {
break
}
}
// 处理最后剩余的缓冲区
if currentStatement.Len() > 0 {
sql := strings.TrimSpace(currentStatement.String())
if sql != "" {
if err := callback(sql); err != nil {
return err
}
}
}
return nil
}

2
server/internal/db/application/db_sql_exec.go
View File

@@ -275,7 +275,7 @@ func (d *dbSqlExecAppImpl) doSelect(ctx context.Context, sqlExecParam *sqlExecPa
case *sqlstmt.SimpleSelectStmt:
qs := stmt.QuerySpecification
limit = qs.Limit
if qs.SelectElements.Star != "" || len(qs.SelectElements.Elements) > 1 {
if qs.SelectElements != nil && (qs.SelectElements.Star != "" || len(qs.SelectElements.Elements) > 1) {
needCheckLimit = true
}
case *sqlstmt.UnionSelectStmt:

2
server/internal/db/dbm/sqlparser/mysql/visitor.go
View File

@@ -279,7 +279,7 @@ func (v *MysqlVisitor) VisitSelectStarElement(ctx *mysqlparser.SelectStarElement
func (v *MysqlVisitor) VisitSelectColumnElement(ctx *mysqlparser.SelectColumnElementContext) interface{} {
sce := new(sqlstmt.SelectColumnElement)
sce.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
sce.FullColumnName = ctx.FullColumnName().Accept(v).(*sqlstmt.ColumnName)
sce.ColumnName = ctx.FullColumnName().Accept(v).(*sqlstmt.ColumnName)
if uid := ctx.Uid(); uid != nil {
sce.Alias = uid.GetText()
}

4
server/internal/db/dbm/sqlparser/pgsql/pgsql_test.go
View File

@@ -9,12 +9,12 @@ import (
func TestParserSimpleSelect(t *testing.T) {
parser := new(PgsqlParser)
sql := `SELECT t.* FROM mayfly.sys_login_log as t where t.id > 0 OFFSET 0 LIMIT 25; select * from tdb where id > 1`
sql := `SELECT t.*,t.id as tid FROM mayfly.sys_login_log as t where t.id > 0 OFFSET 0 LIMIT 25; select * from tdb where id > 1`
stmts, err := parser.Parse(sql)
if err != nil {
t.Fatal(err)
}
stmt := stmts[1].(*sqlstmt.SimpleSelectStmt)
stmt := stmts[0].(*sqlstmt.SimpleSelectStmt)
t.Log(stmt.QuerySpecification.Where.GetText())
fmt.Println(stmt.QuerySpecification.From.GetText())

76
server/internal/db/dbm/sqlparser/pgsql/visitor.go
View File

@@ -166,6 +166,14 @@ func (v *PgsqlVisitor) VisitSimple_select_pramary(ctx *pgparser.Simple_select_pr
qs.From = c.Accept(v).(*sqlstmt.TableSources)
}
if c := ctx.Opt_target_list(); c != nil {
qs.SelectElements = c.Accept(v).(*sqlstmt.SelectElements)
}
if c := ctx.Target_list(); c != nil {
qs.SelectElements = c.Accept(v).(*sqlstmt.SelectElements)
}
if c := ctx.Where_clause(); c != nil {
qs.Where = c.A_expr().Accept(v).(sqlstmt.IExpr)
}
@@ -255,6 +263,74 @@ func (v *PgsqlVisitor) VisitTable_ref(ctx *pgparser.Table_refContext) interface{
return tableSourceBase
}
func (v *PgsqlVisitor) VisitOpt_target_list(ctx *pgparser.Opt_target_listContext) interface{} {
if c := ctx.Target_list(); c != nil {
return c.Accept(v)
}
ses := new(sqlstmt.SelectElements)
ses.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
return ses
}
func (v *PgsqlVisitor) VisitTarget_list(ctx *pgparser.Target_listContext) interface{} {
ses := new(sqlstmt.SelectElements)
ses.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
if tecs := ctx.AllTarget_el(); tecs != nil {
eles := make([]sqlstmt.ISelectElement, 0)
for _, tec := range tecs {
eles = append(eles, tec.Accept(v).(sqlstmt.ISelectElement))
}
ses.Elements = eles
}
if len(ses.Elements) == 1 && ses.Elements[0].GetText() == "*" {
ses.Star = "*"
}
return ses
}
// Visit a parse tree produced by PostgreSQLParser#target_label.
func (v *PgsqlVisitor) VisitTarget_label(ctx *pgparser.Target_labelContext) interface{} {
sce := new(sqlstmt.SelectColumnElement)
sce.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
columnName := new(sqlstmt.ColumnName)
if c := ctx.Collabel(); c != nil {
sce.Alias = c.GetText()
}
if c := ctx.Identifier(); c != nil {
sce.Alias = c.GetText()
}
if exprCtx := ctx.A_expr(); exprCtx != nil {
columnName.Node = sqlstmt.NewNode(ctx.GetParser(), exprCtx)
if aextrCtx := exprCtx.A_expr_qual(); aextrCtx != nil {
col := aextrCtx.GetText()
ownerAndColname := strings.Split(col, ".")
if len(ownerAndColname) == 2 {
columnName.Owner = ownerAndColname[0]
columnName.Identifier = sqlstmt.NewIdentifierValue(ownerAndColname[1])
} else {
columnName.Identifier = sqlstmt.NewIdentifierValue(col)
}
}
} else {
columnName.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
}
sce.ColumnName = columnName
return sce
}
// Visit a parse tree produced by PostgreSQLParser#target_star.
func (v *PgsqlVisitor) VisitTarget_star(ctx *pgparser.Target_starContext) interface{} {
sse := new(sqlstmt.SelectStarElement)
sse.Node = sqlstmt.NewNode(ctx.GetParser(), ctx)
sse.FullId = ctx.STAR().GetText()
return sse
}
func (v *PgsqlVisitor) VisitAlias_clause(ctx *pgparser.Alias_clauseContext) interface{} {
return v.VisitChildren(ctx)
}

118
server/internal/db/dbm/sqlparser/sqlparser.go
View File

@@ -1,12 +1,9 @@
package sqlparser
import (
"bufio"
"bytes"
"io"
"mayfly-go/internal/common/utils"
"mayfly-go/internal/db/dbm/sqlparser/sqlstmt"
"strings"
"unicode/utf8"
)
type DbDialect string
@@ -19,115 +16,6 @@ type SqlParser interface {
}
// SQLSplit sql切割
func SQLSplit(r io.Reader, callback SQLCallback) error {
return parseSQL(r, callback)
}
// SQLCallback 是解析出一条 SQL 语句后的回调函数
type SQLCallback func(sql string) error
// parseSQL 主要由阿里通义灵码提供
func parseSQL(r io.Reader, callback SQLCallback) error {
reader := bufio.NewReaderSize(r, 512*1024)
buffer := new(bytes.Buffer) // 使用 bytes.Buffer 来处理数据
var currentStatement bytes.Buffer
var inString bool
var inMultiLineComment bool
var inSingleLineComment bool
var stringDelimiter rune
var escapeNextChar bool // 用于处理转义符
for {
// 读取数据到缓冲区
data, err := reader.ReadBytes('\n') // 按行读取
if err == io.EOF && len(data) == 0 {
break
}
if err != nil && err != io.EOF {
return err
}
buffer.Write(data)
// 处理缓冲区中的数据
for buffer.Len() > 0 {
r, size := utf8.DecodeRune(buffer.Bytes())
if r == utf8.RuneError && size == 1 {
// 如果解码出错，说明数据不完整，继续读取更多数据
break
}
switch {
case inMultiLineComment:
if r == '*' && buffer.Len() >= 2 && buffer.Bytes()[1] == '/' {
inMultiLineComment = false
buffer.Next(2) // 跳过 '*/'
} else {
buffer.Next(size)
}
case inSingleLineComment:
if r == '\n' {
inSingleLineComment = false
}
buffer.Next(size)
case inString:
if escapeNextChar {
// 当前字符是转义后的字符，直接写入。如后一个为" 避免进入r==stringDelimiter判断被当做字符串结束符中断
currentStatement.WriteRune(r)
escapeNextChar = false
} else if r == '\\' {
// 当前字符是转义符，设置标志位并写入
escapeNextChar = true
currentStatement.WriteRune(r)
} else if r == stringDelimiter {
// 当前字符是字符串结束符，结束字符串处理
inString = false
currentStatement.WriteRune(r)
} else {
// 其他字符，直接写入
currentStatement.WriteRune(r)
}
buffer.Next(size)
case r == '/' && buffer.Len() >= 2 && buffer.Bytes()[1] == '*':
inMultiLineComment = true
buffer.Next(2) // 跳过 '/*'
case r == '-' && buffer.Len() >= 2 && buffer.Bytes()[1] == '-':
inSingleLineComment = true
buffer.Next(2) // 跳过 '--'
case r == '\'' || r == '"':
inString = true
stringDelimiter = r
currentStatement.WriteRune(r)
buffer.Next(size)
case r == ';' && !inString && !inMultiLineComment && !inSingleLineComment:
sql := strings.TrimSpace(currentStatement.String())
if sql != "" {
if err := callback(sql); err != nil {
return err
}
}
currentStatement.Reset()
buffer.Next(size)
default:
currentStatement.WriteRune(r)
buffer.Next(size)
}
}
// 如果读取到 EOF 并且缓冲区为空，退出循环
if err == io.EOF && buffer.Len() == 0 {
break
}
}
// 处理最后剩余的缓冲区
if currentStatement.Len() > 0 {
sql := strings.TrimSpace(currentStatement.String())
if sql != "" {
if err := callback(sql); err != nil {
return err
}
}
}
return nil
func SQLSplit(r io.Reader, callback utils.StmtCallback) error {
return utils.SplitStmts(r, callback)
}

4
server/internal/db/dbm/sqlparser/sqlstmt/select.go
View File

@@ -78,8 +78,8 @@ type (
SelectColumnElement struct {
*Node
FullColumnName *ColumnName
Alias string
ColumnName *ColumnName
Alias string
}
SelectFunctionElement struct {

10
server/internal/redis/application/redis.go
View File

@@ -3,6 +3,7 @@ package application
import (
"context"
"mayfly-go/internal/common/consts"
"mayfly-go/internal/common/utils"
flowapp "mayfly-go/internal/flow/application"
flowentity "mayfly-go/internal/flow/domain/entity"
"mayfly-go/internal/redis/application/dto"
@@ -266,10 +267,10 @@ func (r *redisAppImpl) FlowBizHandle(ctx context.Context, bizHandleParam *flowap
}
handleRes := make([]map[string]any, 0)
cmds := strings.Split(runCmdParam.Cmd, ";\n")
hasErr := false
for _, cmd := range cmds {
cmd = strings.TrimSpace(cmd)
utils.SplitStmts(strings.NewReader(runCmdParam.Cmd), func(stmt string) error {
cmd := strings.TrimSpace(stmt)
runRes := collx.Kvs("cmd", cmd)
if res, err := redisConn.RunCmd(ctx, collx.ArrayMap[string, any](parseRedisCommand(cmd), func(val string) any { return val })...); err != nil {
runRes["res"] = err.Error()
@@ -278,7 +279,8 @@ func (r *redisAppImpl) FlowBizHandle(ctx context.Context, bizHandleParam *flowap
runRes["res"] = res
}
handleRes = append(handleRes, runRes)
}
return nil
})
if hasErr {
return handleRes, errorx.NewBiz("存在命令执行失败")

10
server/internal/redis/application/redis_test.go
View File

@@ -2,10 +2,16 @@ package application
import (
"fmt"
"mayfly-go/internal/common/utils"
"strings"
"testing"
)
func TestParseRedisCommand(t *testing.T) {
res := parseRedisCommand("del 'key l3'")
fmt.Println(res)
utils.SplitStmts(strings.NewReader("del 'key l3'; set key2 key3; set 'key3' 'value3 and value4'; hset field1 key1 'hvalue2 hvalue3'"), func(stmt string) error {
res := parseRedisCommand(stmt)
fmt.Println(res)
return nil
})
}