TRKJ/mayfly-go
2
0
Fork 0
mirror of https://gitee.com/dromara/mayfly-go synced 2025-11-04 00:10:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 数据库、redis、mongo支持ssh隧道等

Browse Source
This commit is contained in:
meilin.huang
2022-07-20 23:25:52 +08:00
parent 802e379f60
commit f0540559bb
30 changed files with 1885 additions and 1694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
server/internal/devops/api/db.go
View File

@@ -56,13 +56,13 @@ func (d *Db) Save(rc *ctx.ReqCtx) {
// 密码脱敏记录日志
form.Password = "****"
if form.Type == "mysql" && form.EnableSSH == 1 {
originSSHPwd, err := utils.DefaultRsaDecrypt(form.SSHPass, true)
biz.ErrIsNilAppendErr(err, "解密密码错误: %s")
db.SSHPass = originSSHPwd
// 密码脱敏记录日志
form.SSHPass = "****"
}
// if form.Type == "mysql" && form.EnableSSH == 1 {
// // originSSHPwd, err := utils.DefaultRsaDecrypt(form.SSHPass, true)
// biz.ErrIsNilAppendErr(err, "解密密码错误: %s")
// // db.SSHPass = originSSHPwd
// // 密码脱敏记录日志
// form.SSHPass = "****"
// }
rc.ReqParam = form

7
server/internal/devops/api/form/db.go
View File

@@ -15,11 +15,8 @@ type DbForm struct {
Env string `json:"env"`
EnvId uint64 `binding:"required" json:"envId"`
EnableSSH int `json:"enable_ssh"`
SSHHost string `json:"ssh_host"`
SSHPort int `json:"ssh_port"`
SSHUser string `json:"ssh_user"`
SSHPass string `json:"ssh_pass"`
EnableSshTunnel int8 `json:"enableSshTunnel"`
SshTunnelMachineId uint64 `json:"sshTunnelMachineId"`
}
type DbSqlSaveForm struct {

14
server/internal/devops/api/form/machine.go
View File

@@ -5,14 +5,12 @@ type MachineForm struct {
ProjectId uint64 `json:"projectId"`
ProjectName string `json:"projectName"`
Name string `json:"name" binding:"required"`
// IP地址
Ip string `json:"ip" binding:"required"`
// 用户名
Username string `json:"username" binding:"required"`
Password string `json:"password"`
// 端口号
Port int `json:"port" binding:"required"`
Remark string `json:"remark"`
Ip string `json:"ip" binding:"required"` // IP地址
Username string `json:"username" binding:"required"` // 用户名
AuthMethod int8 `json:"authMethod" binding:"required"`
Password string `json:"password"`
Port int `json:"port" binding:"required"` // 端口号
Remark string `json:"remark"`
}
type MachineRunForm struct {

16
server/internal/devops/api/form/mongo.go
View File

@@ -1,13 +1,15 @@
package form
type Mongo struct {
Id uint64
Uri string `binding:"required" json:"uri"`
Name string `binding:"required" json:"name"`
ProjectId uint64 `binding:"required" json:"projectId"`
Project string `json:"project"`
Env string `json:"env"`
EnvId uint64 `binding:"required" json:"envId"`
Id uint64
Uri string `binding:"required" json:"uri"`
EnableSshTunnel int8 `json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId uint64 `json:"sshTunnelMachineId"` // ssh隧道机器id
Name string `binding:"required" json:"name"`
ProjectId uint64 `binding:"required" json:"projectId"`
Project string `json:"project"`
Env string `json:"env"`
EnvId uint64 `binding:"required" json:"envId"`
}
type MongoCommand struct {

22
server/internal/devops/api/form/redis.go
View File

@@ -1,16 +1,18 @@
package form
type Redis struct {
Id uint64
Host string `binding:"required" json:"host"`
Password string `json:"password"`
Mode string `json:"mode"`
Db int `json:"db"`
ProjectId uint64 `binding:"required" json:"projectId"`
Project string `json:"project"`
Env string `json:"env"`
EnvId uint64 `binding:"required" json:"envId"`
Remark string `json:"remark"`
Id uint64
Host string `binding:"required" json:"host"`
Password string `json:"password"`
Mode string `json:"mode"`
Db int `json:"db"`
EnableSshTunnel int8 `json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId uint64 `json:"sshTunnelMachineId"` // ssh隧道机器id
ProjectId uint64 `binding:"required" json:"projectId"`
Project string `json:"project"`
Env string `json:"env"`
EnvId uint64 `binding:"required" json:"envId"`
Remark string `json:"remark"`
}
type KeyInfo struct {

18
server/internal/devops/api/machine.go
View File

@@ -54,20 +54,22 @@ func (m *Machine) SaveMachine(rc *ctx.ReqCtx) {
machineForm := new(form.MachineForm)
ginx.BindJsonAndValid(g, machineForm)
entity := new(entity.Machine)
utils.Copy(entity, machineForm)
me := new(entity.Machine)
utils.Copy(me, machineForm)
// 密码解密，并使用解密后的赋值
originPwd, err := utils.DefaultRsaDecrypt(machineForm.Password, true)
biz.ErrIsNilAppendErr(err, "解密密码错误: %s")
entity.Password = originPwd
if me.AuthMethod == entity.MachineAuthMethodPassword {
// 密码解密，并使用解密后的赋值
originPwd, err := utils.DefaultRsaDecrypt(machineForm.Password, true)
biz.ErrIsNilAppendErr(err, "解密密码错误: %s")
me.Password = originPwd
}
// 密码脱敏记录日志
machineForm.Password = "****"
rc.ReqParam = machineForm
entity.SetBaseInfo(rc.LoginAccount)
m.MachineApp.Save(entity)
me.SetBaseInfo(rc.LoginAccount)
m.MachineApp.Save(me)
}
func (m *Machine) ChangeStatus(rc *ctx.ReqCtx) {

6
server/internal/devops/api/vo/db.go
View File

@@ -20,8 +20,6 @@ type SelectDataDbVO struct {
Creator *string `json:"creator"`
CreatorId *int64 `json:"creatorId"`
EnableSSH *int `json:"enable_ssh"`
SSHHost *string `json:"ssh_host"`
SSHPort *int `json:"ssh_port"`
SSHUser *string `json:"ssh_user"`
EnableSshTunnel *int8 `json:"enableSshTunnel"`
SshTunnelMachineId *uint64 `json:"sshTunnelMachineId"`
}

24
server/internal/devops/api/vo/redis.go
View File

@@ -5,17 +5,19 @@ import "time"
type Redis struct {
Id *int64 `json:"id"`
// Name *string `json:"name"`
Host *string `json:"host"`
Db int `json:"db"`
ProjectId *int64 `json:"projectId"`
Project *string `json:"project"`
Mode *string `json:"mode"`
Remark *string `json:"remark"`
Env *string `json:"env"`
EnvId *int64 `json:"envId"`
CreateTime *time.Time `json:"createTime"`
Creator *string `json:"creator"`
CreatorId *int64 `json:"creatorId"`
Host *string `json:"host"`
Db int `json:"db"`
ProjectId *int64 `json:"projectId"`
Project *string `json:"project"`
Mode *string `json:"mode"`
EnableSshTunnel *int8 `orm:"column(enable_ssh_tunnel)" json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId *uint64 `orm:"column(ssh_tunnel_machine_id)" json:"sshTunnelMachineId"` // ssh隧道机器id
Remark *string `json:"remark"`
Env *string `json:"env"`
EnvId *int64 `json:"envId"`
CreateTime *time.Time `json:"createTime"`
Creator *string `json:"creator"`
CreatorId *int64 `json:"creatorId"`
}
type Keys struct {

1
server/internal/devops/api/vo/vo.go
View File

@@ -22,6 +22,7 @@ type MachineVO struct {
Username *string `json:"username"`
Ip *string `json:"ip"`
Port *int `json:"port"`
AuthMethod *int8 `json:"authMethod"`
Status *int8 `json:"status"`
CreateTime *time.Time `json:"createTime"`
Creator *string `json:"creator"`

137
server/internal/devops/application/db_app.go
View File

@@ -1,11 +1,13 @@
package application
import (
"context"
"database/sql"
"errors"
"fmt"
"mayfly-go/internal/devops/domain/entity"
"mayfly-go/internal/devops/domain/repository"
"mayfly-go/internal/devops/infrastructure/machine"
"mayfly-go/internal/devops/infrastructure/persistence"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/cache"
@@ -20,7 +22,8 @@ import (
"time"
"github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/lib/pq"
"golang.org/x/crypto/ssh"
)
type Db interface {
@@ -77,15 +80,11 @@ func (d *dbAppImpl) GetById(id uint64, cols ...string) *entity.Db {
func (d *dbAppImpl) Save(dbEntity *entity.Db) {
// 默认tcp连接
if dbEntity.Type == "mysql" && dbEntity.EnableSSH == 1 {
dbEntity.Network = "mysql+ssh"
} else {
dbEntity.Network = "tcp"
}
dbEntity.Network = dbEntity.GetNetwork()
// 测试连接
if dbEntity.Password != "" {
TestConnection(*dbEntity)
TestConnection(dbEntity)
}
// 查找是否存在该库
@@ -109,6 +108,8 @@ func (d *dbAppImpl) Save(dbEntity *entity.Db) {
var oldDbs []interface{}
for _, v := range strings.Split(old.Database, " ") {
// 关闭数据库连接
CloseDb(dbEntity.Id, v)
oldDbs = append(oldDbs, v)
}
@@ -121,14 +122,11 @@ func (d *dbAppImpl) Save(dbEntity *entity.Db) {
return i1.(string) == i2.(string)
})
for _, v := range delDb {
// 先关闭数据库连接
CloseDb(dbEntity.Id, v.(string))
// 删除该库关联的所有sql记录
d.dbSqlRepo.DeleteBy(&entity.DbSql{DbId: dbId, Db: v.(string)})
}
d.dbRepo.Update(dbEntity)
}
func (d *dbAppImpl) Delete(id uint64) {
@@ -160,28 +158,43 @@ func (da *dbAppImpl) GetDbInstance(id uint64, db string) *DbInstance {
d := da.GetById(id)
biz.NotNil(d, "数据库信息不存在")
biz.IsTrue(strings.Contains(d.Database, db), "未配置该库的操作权限")
global.Log.Infof("连接db: %s:%d/%s", d.Host, d.Port, db)
cacheKey := GetDbCacheKey(id, db)
dbi := &DbInstance{Id: cacheKey, Type: d.Type, ProjectId: d.ProjectId}
//SSH Conect
if d.Type == "mysql" && d.EnableSSH == 1 {
sshClient, err := utils.SSHConnect(d.SSHUser, d.SSHPass, d.SSHHost, "", d.SSHPort)
if err != nil {
global.Log.Errorf("ssh连接失败: %s@%s:%d", d.SSHUser, d.SSHHost, d.SSHPort)
panic(biz.NewBizErr(fmt.Sprintf("ssh连接失败: %s", err.Error())))
if d.EnableSshTunnel == 1 && d.SshTunnelMachineId != 0 {
me := MachineApp.GetById(d.SshTunnelMachineId)
biz.NotNil(me, "隧道机器信息不存在")
sshClient, err := machine.GetSshClient(me)
biz.ErrIsNilAppendErr(err, "ssh隧道连接失败: %s")
dbi.sshTunnel = sshClient
if d.Type == entity.DbTypeMysql {
mysql.RegisterDialContext(d.Network, func(ctx context.Context, addr string) (net.Conn, error) {
return sshClient.Dial("tcp", addr)
})
} else if d.Type == entity.DbTypePostgres {
_, err := pq.DialOpen(&PqSqlDialer{sshTunnel: sshClient}, getDsn(d))
if err != nil {
dbi.Close()
panic(biz.NewBizErr(fmt.Sprintf("postgres隧道连接失败: %s", err.Error())))
}
}
mysql.RegisterDial("mysql+ssh", func(addr string) (net.Conn, error) {
return sshClient.Dial("tcp", addr)
})
}
// 将数据库替换为要访问的数据库，原本数据库为空格拼接的所有库
d.Database = db
DB, err := sql.Open(d.Type, getDsn(d))
biz.ErrIsNil(err, fmt.Sprintf("Open %s failed, err:%v\n", d.Type, err))
perr := DB.Ping()
if perr != nil {
if err != nil {
dbi.Close()
panic(biz.NewBizErr(fmt.Sprintf("Open %s failed, err:%v\n", d.Type, err)))
}
err = DB.Ping()
if err != nil {
dbi.Close()
global.Log.Errorf("连接db失败: %s:%d/%s", d.Host, d.Port, db)
panic(biz.NewBizErr(fmt.Sprintf("数据库连接失败: %s", perr.Error())))
panic(biz.NewBizErr(fmt.Sprintf("数据库连接失败: %s", err.Error())))
}
// 最大连接周期，超过时间的连接就close
@@ -191,14 +204,30 @@ func (da *dbAppImpl) GetDbInstance(id uint64, db string) *DbInstance {
// 设置闲置连接数
DB.SetMaxIdleConns(1)
cacheKey := GetDbCacheKey(id, db)
dbi := &DbInstance{Id: cacheKey, Type: d.Type, ProjectId: d.ProjectId, db: DB}
dbi.db = DB
global.Log.Infof("连接db: %s:%d/%s", d.Host, d.Port, db)
if needCache {
dbCache.Put(cacheKey, dbi)
}
return dbi
}
type PqSqlDialer struct {
sshTunnel *ssh.Client
}
func (pd *PqSqlDialer) Dial(network, address string) (net.Conn, error) {
if sshConn, err := pd.sshTunnel.Dial(network, address); err == nil {
// 将ssh conn包装，否则redis内部设置超时会报错,ssh conn不支持设置超时会返回错误: ssh: tcpChan: deadline not supported
return &utils.WrapSshConn{Conn: sshConn}, nil
} else {
return nil, err
}
}
func (pd *PqSqlDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
return pd.Dial(network, address)
}
//------------------------------------------------------------------------------
// 客户端连接缓存，30分钟内没有访问则会被关闭, key为数据库实例id:数据库
@@ -220,22 +249,28 @@ func GetDbInstanceByCache(id string) *DbInstance {
return nil
}
func TestConnection(d entity.Db) {
func TestConnection(d *entity.Db) {
//SSH Conect
if d.Type == "mysql" && d.EnableSSH == 1 {
sshClient, err := utils.SSHConnect(d.SSHUser, d.SSHPass, d.SSHHost, "", d.SSHPort)
if err != nil {
global.Log.Errorf("ssh连接失败: %s@%s:%d", d.SSHUser, d.SSHHost, d.SSHPort)
panic(biz.NewBizErr(fmt.Sprintf("ssh连接失败: %s", err.Error())))
if d.EnableSshTunnel == 1 && d.SshTunnelMachineId != 0 {
me := MachineApp.GetById(d.SshTunnelMachineId)
sshClient, err := machine.GetSshClient(me)
biz.ErrIsNilAppendErr(err, "ssh隧道连接失败: %s")
defer sshClient.Close()
if d.Type == entity.DbTypeMysql {
mysql.RegisterDialContext(d.Network, func(ctx context.Context, addr string) (net.Conn, error) {
return sshClient.Dial("tcp", addr)
})
} else if d.Type == entity.DbTypePostgres {
_, err := pq.DialOpen(&PqSqlDialer{sshTunnel: sshClient}, getDsn(d))
if err != nil {
panic(biz.NewBizErr(fmt.Sprintf("postgres隧道连接失败: %s", err.Error())))
}
}
mysql.RegisterDial("mysql+ssh", func(addr string) (net.Conn, error) {
return sshClient.Dial("tcp", addr)
})
}
// 验证第一个库是否可以连接即可
d.Database = strings.Split(d.Database, " ")[0]
DB, err := sql.Open(d.Type, getDsn(&d))
DB, err := sql.Open(d.Type, getDsn(d))
biz.ErrIsNil(err, "Open %s failed, err:%v\n", d.Type, err)
defer DB.Close()
perr := DB.Ping()
@@ -248,6 +283,7 @@ type DbInstance struct {
Type string
ProjectId uint64
db *sql.DB
sshTunnel *ssh.Client
}
// 执行查询语句
@@ -359,13 +395,22 @@ func (d *DbInstance) Exec(sql string) (int64, error) {
// 关闭连接
func (d *DbInstance) Close() {
d.db.Close()
if d.db != nil {
if err := d.db.Close(); err != nil {
global.Log.Errorf("关闭数据库实例[%s]连接失败: %s", d.Id, err.Error())
}
}
if d.sshTunnel != nil {
if err := d.sshTunnel.Close(); err != nil {
global.Log.Errorf("关闭数据库实例[%s]的ssh隧道失败: %s", d.Id, err.Error())
}
}
}
// 获取dataSourceName
func getDsn(d *entity.Db) string {
var dsn string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
dsn = fmt.Sprintf("%s:%s@%s(%s:%d)/%s?timeout=8s", d.Username, d.Password, d.Network, d.Host, d.Port, d.Database)
if d.Params != "" {
dsn = fmt.Sprintf("%s&%s", dsn, d.Params)
@@ -373,7 +418,7 @@ func getDsn(d *entity.Db) string {
return dsn
}
if d.Type == "postgres" {
if d.Type == entity.DbTypePostgres {
dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", d.Host, d.Port, d.Username, d.Password, d.Database)
if d.Params != "" {
dsn = fmt.Sprintf("%s %s", dsn, strings.Join(strings.Split(d.Params, "&"), " "))
@@ -469,7 +514,7 @@ const (
func (d *DbInstance) GetTableMetedatas() []map[string]interface{} {
var sql string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
sql = MYSQL_TABLE_MA
} else if d.Type == "postgres" {
sql = PGSQL_TABLE_MA
@@ -489,10 +534,10 @@ func (d *DbInstance) GetColumnMetadatas(tableNames ...string) []map[string]inter
var countSqlTmp string
var sqlTmp string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
countSqlTmp = MYSQL_COLOUMN_MA_COUNT
sqlTmp = MYSQL_COLUMN_MA
} else if d.Type == "postgres" {
} else if d.Type == entity.DbTypePostgres {
countSqlTmp = PGSQL_COLUMN_MA_COUNT
sqlTmp = PGSQL_COLUMN_MA
}
@@ -524,9 +569,9 @@ func (d *DbInstance) GetPrimaryKey(tablename string) string {
func (d *DbInstance) GetTableInfos() []map[string]interface{} {
var sql string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
sql = MYSQL_TABLE_INFO
} else if d.Type == "postgres" {
} else if d.Type == entity.DbTypePostgres {
sql = PGSQL_TABLE_INFO
}
_, res, _ := d.SelectData(sql)
@@ -535,9 +580,9 @@ func (d *DbInstance) GetTableInfos() []map[string]interface{} {
func (d *DbInstance) GetTableIndex(tableName string) []map[string]interface{} {
var sql string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
sql = fmt.Sprintf(MYSQL_INDEX_INFO, tableName)
} else if d.Type == "postgres" {
} else if d.Type == entity.DbTypePostgres {
sql = fmt.Sprintf(PGSQL_INDEX_INFO, tableName)
}
_, res, _ := d.SelectData(sql)
@@ -546,7 +591,7 @@ func (d *DbInstance) GetTableIndex(tableName string) []map[string]interface{} {
func (d *DbInstance) GetCreateTableDdl(tableName string) []map[string]interface{} {
var sql string
if d.Type == "mysql" {
if d.Type == entity.DbTypeMysql {
sql = fmt.Sprintf("show create table %s ", tableName)
}
_, res, _ := d.SelectData(sql)

86
server/internal/devops/application/mongo_app.go
View File

@@ -4,15 +4,19 @@ import (
"context"
"mayfly-go/internal/devops/domain/entity"
"mayfly-go/internal/devops/domain/repository"
"mayfly-go/internal/devops/infrastructure/machine"
"mayfly-go/internal/devops/infrastructure/persistence"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/cache"
"mayfly-go/pkg/global"
"mayfly-go/pkg/model"
"mayfly-go/pkg/utils"
"net"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"golang.org/x/crypto/ssh"
)
type Mongo interface {
@@ -80,13 +84,13 @@ func (d *mongoAppImpl) Save(m *entity.Mongo) {
}
func (d *mongoAppImpl) GetMongoCli(id uint64) *mongo.Client {
cli, err := GetMongoCli(id, func(u uint64) string {
mongo := d.GetById(id)
mongoInstance, err := GetMongoInstance(id, func(u uint64) *entity.Mongo {
mongo := d.GetById(u)
biz.NotNil(mongo, "mongo信息不存在")
return mongo.Uri
return mongo
})
biz.ErrIsNilAppendErr(err, "连接mongo失败: %s")
return cli
return mongoInstance.Cli
}
// -----------------------------------------------------------
@@ -95,21 +99,22 @@ func (d *mongoAppImpl) GetMongoCli(id uint64) *mongo.Client {
var mongoCliCache = cache.NewTimedCache(30*time.Minute, 5*time.Second).
WithUpdateAccessTime(true).
OnEvicted(func(key interface{}, value interface{}) {
global.Log.Info("关闭mongo连接: id = ", key)
value.(*mongo.Client).Disconnect(context.TODO())
global.Log.Info("删除mongo连接缓存: id = ", key)
value.(*MongoInstance).Close()
})
func GetMongoCli(mongoId uint64, getMongoUri func(uint64) string) (*mongo.Client, error) {
cli, err := mongoCliCache.ComputeIfAbsent(mongoId, func(key interface{}) (interface{}, error) {
c, err := connect(getMongoUri(mongoId))
// 获取mongo的连接实例
func GetMongoInstance(mongoId uint64, getMongoEntity func(uint64) *entity.Mongo) (*MongoInstance, error) {
mi, err := mongoCliCache.ComputeIfAbsent(mongoId, func(_ interface{}) (interface{}, error) {
c, err := connect(getMongoEntity(mongoId))
if err != nil {
return nil, err
}
return c, nil
})
if cli != nil {
return cli.(*mongo.Client), err
if mi != nil {
return mi.(*MongoInstance), err
}
return nil, err
}
@@ -118,16 +123,67 @@ func DeleteMongoCache(mongoId uint64) {
mongoCliCache.Delete(mongoId)
}
type MongoInstance struct {
Id uint64
ProjectId uint64
Cli *mongo.Client
sshTunnel *ssh.Client
}
func (mi *MongoInstance) Close() {
if mi.Cli != nil {
if err := mi.Cli.Disconnect(context.Background()); err != nil {
global.Log.Errorf("关闭mongo实例[%d]连接失败: %s", mi.Id, err)
}
}
if mi.sshTunnel != nil {
if err := mi.sshTunnel.Close(); err != nil {
global.Log.Errorf("关闭mongo实例[%d]的ssh隧道失败: %s", mi.Id, err.Error())
}
}
}
// 连接mongo，并返回client
func connect(uri string) (*mongo.Client, error) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
func connect(me *entity.Mongo) (*MongoInstance, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(2))
mongoInstance := &MongoInstance{Id: me.Id, ProjectId: me.ProjectId}
mongoOptions := options.Client().ApplyURI(me.Uri).
SetMaxPoolSize(1)
// 启用ssh隧道则连接隧道机器
if me.EnableSshTunnel == 1 {
machineEntity := MachineApp.GetById(4)
sshClient, err := machine.GetSshClient(machineEntity)
biz.ErrIsNilAppendErr(err, "ssh隧道连接失败: %s")
mongoInstance.sshTunnel = sshClient
mongoOptions.SetDialer(&MongoSshDialer{sshTunnel: sshClient})
}
client, err := mongo.Connect(ctx, mongoOptions)
if err != nil {
return nil, err
}
if err = client.Ping(context.TODO(), nil); err != nil {
return nil, err
}
return client, err
global.Log.Infof("连接mongo: %s", me.Uri)
mongoInstance.Cli = client
return mongoInstance, err
}
type MongoSshDialer struct {
sshTunnel *ssh.Client
}
func (sd *MongoSshDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
if sshConn, err := sd.sshTunnel.Dial(network, address); err == nil {
// 将ssh conn包装，否则内部部设置超时会报错,ssh conn不支持设置超时会返回错误: ssh: tcpChan: deadline not supported
return &utils.WrapSshConn{Conn: sshConn}, nil
} else {
return nil, err
}
}

94
server/internal/devops/application/redis_app.go
View File

@@ -5,15 +5,19 @@ import (
"fmt"
"mayfly-go/internal/devops/domain/entity"
"mayfly-go/internal/devops/domain/repository"
"mayfly-go/internal/devops/infrastructure/machine"
"mayfly-go/internal/devops/infrastructure/persistence"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/cache"
"mayfly-go/pkg/global"
"mayfly-go/pkg/model"
"mayfly-go/pkg/utils"
"net"
"strings"
"time"
"github.com/go-redis/redis/v8"
"golang.org/x/crypto/ssh"
)
type Redis interface {
@@ -109,25 +113,23 @@ func (r *redisAppImpl) GetRedisInstance(id uint64) *RedisInstance {
biz.NotNil(re, "redis信息不存在")
redisMode := re.Mode
ri := &RedisInstance{Id: id, ProjectId: re.ProjectId, Mode: redisMode}
var ri *RedisInstance
if redisMode == "" || redisMode == entity.RedisModeStandalone {
rcli := getRedisCient(re)
ri = getRedisCient(re)
// 测试连接
_, e := rcli.Ping(context.Background()).Result()
_, e := ri.Cli.Ping(context.Background()).Result()
if e != nil {
rcli.Close()
ri.Close()
panic(biz.NewBizErr(fmt.Sprintf("redis连接失败: %s", e.Error())))
}
ri.Cli = rcli
} else if redisMode == entity.RedisModeCluster {
ccli := getRedisClusterClient(re)
ri = getRedisClusterClient(re)
// 测试连接
_, e := ccli.Ping(context.Background()).Result()
_, e := ri.ClusterCli.Ping(context.Background()).Result()
if e != nil {
ccli.Close()
ri.Close()
panic(biz.NewBizErr(fmt.Sprintf("redis集群连接失败: %s", e.Error())))
}
ri.ClusterCli = ccli
}
global.Log.Infof("连接redis: %s", re.Host)
@@ -137,21 +139,56 @@ func (r *redisAppImpl) GetRedisInstance(id uint64) *RedisInstance {
return ri
}
func getRedisCient(re *entity.Redis) *redis.Client {
return redis.NewClient(&redis.Options{
Addr: re.Host,
Password: re.Password, // no password set
DB: re.Db, // use default DB
DialTimeout: 8 * time.Second,
})
func getRedisCient(re *entity.Redis) *RedisInstance {
ri := &RedisInstance{Id: re.Id, ProjectId: re.ProjectId, Mode: re.Mode}
redisOptions := &redis.Options{
Addr: re.Host,
Password: re.Password, // no password set
DB: re.Db, // use default DB
DialTimeout: 8 * time.Second,
ReadTimeout: -1, // Disable timeouts, because SSH does not support deadlines.
WriteTimeout: -1,
}
if re.EnableSshTunnel == 1 {
sshClient, dialerFunc := getRedisDialer(re.SshTunnelMachineId)
ri.sshTunnel = sshClient
redisOptions.Dialer = dialerFunc
}
ri.Cli = redis.NewClient(redisOptions)
return ri
}
func getRedisClusterClient(re *entity.Redis) *redis.ClusterClient {
return redis.NewClusterClient(&redis.ClusterOptions{
func getRedisClusterClient(re *entity.Redis) *RedisInstance {
ri := &RedisInstance{Id: re.Id, ProjectId: re.ProjectId, Mode: re.Mode}
redisClusterOptions := &redis.ClusterOptions{
Addrs: strings.Split(re.Host, ","),
Password: re.Password,
DialTimeout: 8 * time.Second,
})
}
if re.EnableSshTunnel == 1 {
sshClient, dialerFunc := getRedisDialer(re.SshTunnelMachineId)
ri.sshTunnel = sshClient
redisClusterOptions.Dialer = dialerFunc
}
ri.ClusterCli = redis.NewClusterClient(redisClusterOptions)
return ri
}
func getRedisDialer(machineId uint64) (*ssh.Client, func(ctx context.Context, network, addr string) (net.Conn, error)) {
me := MachineApp.GetById(machineId)
sshClient, err := machine.GetSshClient(me)
biz.ErrIsNilAppendErr(err, "ssh隧道连接失败: %s")
return sshClient, func(_ context.Context, network, addr string) (net.Conn, error) {
if sshConn, err := sshClient.Dial(network, addr); err == nil {
// 将ssh conn包装，否则redis内部设置超时会报错,ssh conn不支持设置超时会返回错误: ssh: tcpChan: deadline not supported
return &utils.WrapSshConn{Conn: sshConn}, nil
} else {
return nil, err
}
}
}
//------------------------------------------------------------------------------
@@ -174,11 +211,11 @@ func TestRedisConnection(re *entity.Redis) {
if re.Mode == "" || re.Mode == entity.RedisModeStandalone {
rcli := getRedisCient(re)
defer rcli.Close()
cmd = rcli
cmd = rcli.Cli
} else if re.Mode == entity.RedisModeCluster {
ccli := getRedisClusterClient(re)
defer ccli.Close()
cmd = ccli
cmd = ccli.ClusterCli
}
// 测试连接
@@ -193,6 +230,7 @@ type RedisInstance struct {
Mode string
Cli *redis.Client
ClusterCli *redis.ClusterClient
sshTunnel *ssh.Client
}
// 获取命令执行接口的具体实现
@@ -215,10 +253,18 @@ func (r *RedisInstance) Scan(cursor uint64, match string, count int64) ([]string
func (r *RedisInstance) Close() {
if r.Mode == entity.RedisModeStandalone {
r.Cli.Close()
return
if err := r.Cli.Close(); err != nil {
global.Log.Errorf("关闭redis单机实例[%d]连接失败: %s", r.Id, err.Error())
}
}
if r.Mode == entity.RedisModeCluster {
r.ClusterCli.Close()
if err := r.ClusterCli.Close(); err != nil {
global.Log.Errorf("关闭redis集群实例[%d]连接失败: %s", r.Id, err.Error())
}
}
if r.sshTunnel != nil {
if err := r.sshTunnel.Close(); err != nil {
global.Log.Errorf("关闭redis实例[%d]的ssh隧道失败: %s", r.Id, err.Error())
}
}
}

26
server/internal/devops/domain/entity/db.go
View File

@@ -1,6 +1,7 @@
package entity
import (
"fmt"
"mayfly-go/pkg/model"
)
@@ -21,9 +22,24 @@ type Db struct {
EnvId uint64
Env string
EnableSSH int `orm:"column(enable_ssh)" json:"enable_ssh"`
SSHHost string `orm:"column(ssh_host)" json:"ssh_host"`
SSHPort int `orm:"column(ssh_port)" json:"ssh_port"`
SSHUser string `orm:"column(ssh_user)" json:"ssh_user"`
SSHPass string `orm:"column(ssh_pass)" json:"-"`
EnableSshTunnel int8 `orm:"column(enable_ssh_tunnel)" json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId uint64 `orm:"column(ssh_tunnel_machine_id)" json:"sshTunnelMachineId"` // ssh隧道机器id
}
// 获取数据库连接网络, 若没有使用ssh隧道，则直接返回。否则返回拼接的网络需要注册至指定dial
func (d Db) GetNetwork() string {
network := d.Network
if d.EnableSshTunnel == -1 {
if network == "" {
return "tcp"
} else {
return network
}
}
return fmt.Sprintf("%s+ssh:%d", d.Type, d.SshTunnelMachineId)
}
const (
DbTypeMysql = "mysql"
DbTypePostgres = "postgres"
)

11
server/internal/devops/domain/entity/machine.go
View File

@@ -9,8 +9,9 @@ type Machine struct {
ProjectId uint64 `json:"projectId"`
ProjectName string `json:"projectName"`
Name string `json:"name"`
Ip string `json:"ip"` // IP地址
Username string `json:"username"` // 用户名
Ip string `json:"ip"` // IP地址
Username string `json:"username"` // 用户名
AuthMethod int8 `json:"authMethod"` // 授权认证方式
Password string `json:"-"`
Port int `json:"port"` // 端口号
Status int8 `json:"status"` // 状态 1:启用；2:停用
@@ -18,6 +19,8 @@ type Machine struct {
}
const (
MachineStatusEnable int8 = 1 // 启用状态
MachineStatusDisable int8 = -1 // 禁用状态
MachineStatusEnable int8 = 1 // 启用状态
MachineStatusDisable int8 = -1 // 禁用状态
MachineAuthMethodPassword int8 = 1 // 密码登录
MachineAuthMethodPublicKey int8 = 2 // 公钥免密登录
)

14
server/internal/devops/domain/entity/mongo.go
View File

@@ -5,10 +5,12 @@ import "mayfly-go/pkg/model"
type Mongo struct {
model.Model
Name string `orm:"column(name)" json:"name"`
Uri string `orm:"column(uri)" json:"uri"`
ProjectId uint64 `json:"projectId"`
Project string `json:"project"`
EnvId uint64 `json:"envId"`
Env string `json:"env"`
Name string `orm:"column(name)" json:"name"`
Uri string `orm:"column(uri)" json:"uri"`
EnableSshTunnel int8 `orm:"column(enable_ssh_tunnel)" json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId uint64 `orm:"column(ssh_tunnel_machine_id)" json:"sshTunnelMachineId"` // ssh隧道机器id
ProjectId uint64 `json:"projectId"`
Project string `json:"project"`
EnvId uint64 `json:"envId"`
Env string `json:"env"`
}

20
server/internal/devops/domain/entity/redis.go
View File

@@ -7,15 +7,17 @@ import (
type Redis struct {
model.Model
Host string `orm:"column(host)" json:"host"`
Mode string `json:"mode"`
Password string `orm:"column(password)" json:"-"`
Db int `orm:"column(database)" json:"db"`
Remark string
ProjectId uint64
Project string
EnvId uint64
Env string
Host string `orm:"column(host)" json:"host"`
Mode string `json:"mode"`
Password string `orm:"column(password)" json:"-"`
Db int `orm:"column(database)" json:"db"`
EnableSshTunnel int8 `orm:"column(enable_ssh_tunnel)" json:"enableSshTunnel"` // 是否启用ssh隧道
SshTunnelMachineId uint64 `orm:"column(ssh_tunnel_machine_id)" json:"sshTunnelMachineId"` // ssh隧道机器id
Remark string
ProjectId uint64
Project string
EnvId uint64
Env string
}
const (

170
server/internal/devops/infrastructure/machine/machine.go
View File

@@ -24,58 +24,6 @@ type Cli struct {
sftpClient *sftp.Client
}
// 机器客户端连接缓存，45分钟内没有访问则会被关闭
var cliCache = cache.NewTimedCache(45*time.Minute, 5*time.Second).
WithUpdateAccessTime(true).
OnEvicted(func(key interface{}, value interface{}) {
value.(*Cli).Close()
})
// 是否存在指定id的客户端连接
func HasCli(machineId uint64) bool {
if _, ok := cliCache.Get(machineId); ok {
return true
}
return false
}
// 删除指定机器客户端，并关闭客户端连接
func DeleteCli(id uint64) {
cliCache.Delete(id)
}
// 从缓存中获取客户端信息，不存在则回调获取机器信息函数，并新建
func GetCli(machineId uint64, getMachine func(uint64) *entity.Machine) (*Cli, error) {
cli, err := cliCache.ComputeIfAbsent(machineId, func(key interface{}) (interface{}, error) {
c, err := newClient(getMachine(machineId))
if err != nil {
return nil, err
}
return c, nil
})
if cli != nil {
return cli.(*Cli), err
}
return nil, err
}
//根据机器信息创建客户端对象
func newClient(machine *entity.Machine) (*Cli, error) {
if machine == nil {
return nil, errors.New("机器不存在")
}
global.Log.Infof("[%s]机器连接：%s:%d", machine.Name, machine.Ip, machine.Port)
cli := new(Cli)
cli.machine = machine
err := cli.connect()
if err != nil {
return nil, err
}
return cli, nil
}
//连接
func (c *Cli) connect() error {
// 如果已经有client则直接返回
@@ -83,16 +31,7 @@ func (c *Cli) connect() error {
return nil
}
m := c.machine
config := ssh.ClientConfig{
User: m.Username,
Auth: []ssh.AuthMethod{ssh.Password(m.Password)},
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
Timeout: 5 * time.Second,
}
addr := fmt.Sprintf("%s:%d", m.Ip, m.Port)
sshClient, err := ssh.Dial("tcp", addr, &config)
sshClient, err := GetSshClient(m)
if err != nil {
return err
}
@@ -100,25 +39,6 @@ func (c *Cli) connect() error {
return nil
}
// 测试连接
func TestConn(m *entity.Machine) error {
config := ssh.ClientConfig{
User: m.Username,
Auth: []ssh.AuthMethod{ssh.Password(m.Password)},
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
Timeout: 5 * time.Second,
}
addr := fmt.Sprintf("%s:%d", m.Ip, m.Port)
sshClient, err := ssh.Dial("tcp", addr, &config)
if err != nil {
return err
}
defer sshClient.Close()
return nil
}
// 关闭client和并从缓存中移除
func (c *Cli) Close() {
m := c.machine
@@ -184,3 +104,91 @@ func (c *Cli) Run(shell string) (*string, error) {
func (c *Cli) GetMachine() *entity.Machine {
return c.machine
}
// 机器客户端连接缓存，45分钟内没有访问则会被关闭
var cliCache = cache.NewTimedCache(45*time.Minute, 5*time.Second).
WithUpdateAccessTime(true).
OnEvicted(func(_, value interface{}) {
value.(*Cli).Close()
})
// 是否存在指定id的客户端连接
func HasCli(machineId uint64) bool {
if _, ok := cliCache.Get(machineId); ok {
return true
}
return false
}
// 删除指定机器客户端，并关闭客户端连接
func DeleteCli(id uint64) {
cliCache.Delete(id)
}
// 从缓存中获取客户端信息，不存在则回调获取机器信息函数，并新建
func GetCli(machineId uint64, getMachine func(uint64) *entity.Machine) (*Cli, error) {
cli, err := cliCache.ComputeIfAbsent(machineId, func(_ interface{}) (interface{}, error) {
c, err := newClient(getMachine(machineId))
if err != nil {
return nil, err
}
return c, nil
})
if cli != nil {
return cli.(*Cli), err
}
return nil, err
}
// 测试连接
func TestConn(m *entity.Machine) error {
sshClient, err := GetSshClient(m)
if err != nil {
return err
}
defer sshClient.Close()
return nil
}
func GetSshClient(m *entity.Machine) (*ssh.Client, error) {
config := ssh.ClientConfig{
User: m.Username,
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
Timeout: 5 * time.Second,
}
if m.AuthMethod == entity.MachineAuthMethodPassword {
config.Auth = []ssh.AuthMethod{ssh.Password(m.Password)}
} else if m.AuthMethod == entity.MachineAuthMethodPublicKey {
if signer, err := ssh.ParsePrivateKey([]byte(m.Password)); err != nil {
return nil, err
} else {
config.Auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
}
}
addr := fmt.Sprintf("%s:%d", m.Ip, m.Port)
sshClient, err := ssh.Dial("tcp", addr, &config)
if err != nil {
return nil, err
}
return sshClient, nil
}
//根据机器信息创建客户端对象
func newClient(machine *entity.Machine) (*Cli, error) {
if machine == nil {
return nil, errors.New("机器不存在")
}
global.Log.Infof("[%s]机器连接：%s:%d", machine.Name, machine.Ip, machine.Port)
cli := new(Cli)
cli.machine = machine
err := cli.connect()
if err != nil {
return nil, err
}
return cli, nil
}