DNS节点增加在线状态通知

internal/db/models/message_dao.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -28,8 +29,8 @@ const (
 	MessageTypeHealthCheckFailed          MessageType = "HealthCheckFailed"          // 节点健康检查失败
 	MessageTypeHealthCheckNodeUp          MessageType = "HealthCheckNodeUp"          // 因健康检查节点上线
 	MessageTypeHealthCheckNodeDown        MessageType = "HealthCheckNodeDown"        // 因健康检查节点下线
-	MessageTypeNodeInactive               MessageType = "NodeInactive"               // 节点不活跃
+	MessageTypeNodeInactive               MessageType = "NodeInactive"               // 边缘节点不活跃
-	MessageTypeNodeActive                 MessageType = "NodeActive"                 // 节点活跃
+	MessageTypeNodeActive                 MessageType = "NodeActive"                 // 边缘节点活跃
 	MessageTypeClusterDNSSyncFailed       MessageType = "ClusterDNSSyncFailed"       // DNS同步失败
 	MessageTypeSSLCertExpiring            MessageType = "SSLCertExpiring"            // SSL证书即将过期
 	MessageTypeSSLCertACMETaskFailed      MessageType = "SSLCertACMETaskFailed"      // SSL证书任务执行失败
@@ -39,6 +40,9 @@ const (
 	MessageTypeServerNamesAuditingFailed  MessageType = "ServerNamesAuditingFailed"  // 服务域名审核失败
 	MessageTypeThresholdSatisfied         MessageType = "ThresholdSatisfied"         // 满足阈值
 	MessageTypeFirewallEvent              MessageType = "FirewallEvent"              // 防火墙事件
+
+	MessageTypeNSNodeInactive MessageType = "NSNodeInactive" // 边缘节点不活跃
+	MessageTypeNSNodeActive   MessageType = "NSNodeActive"   // 边缘节点活跃
 )
 
 type MessageDAO dbs.DAO
@@ -93,8 +97,8 @@ func (this *MessageDAO) FindEnabledMessage(tx *dbs.Tx, id int64) (*Message, erro
 }
 
 // CreateClusterMessage 创建集群消息
-func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, clusterId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
+func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, role string, clusterId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
-	_, err := this.createMessage(tx, clusterId, 0, messageType, level, subject, body, paramsJSON)
+	_, err := this.createMessage(tx, role, clusterId, 0, messageType, level, subject, body, paramsJSON)
 	if err != nil {
 		return err
 	}
@@ -113,9 +117,9 @@ func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, clusterId int64, messag
 }
 
 // CreateNodeMessage 创建节点消息
-func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
+func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, role string, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
 	// 检查N分钟内是否已经发送过
-	hash := this.calHash(subject, body, paramsJSON)
+	hash := this.calHash(role, clusterId, nodeId, subject, body, paramsJSON)
 	exists, err := this.Query(tx).
 		Attr("hash", hash).
 		Gt("createdAt", time.Now().Unix()-10*60). // 10分钟
@@ -127,11 +131,13 @@ func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, clusterId int64, nodeId in
 		return nil
 	}
 
-	_, err = this.createMessage(tx, clusterId, nodeId, messageType, level, subject, body, paramsJSON)
+	_, err = this.createMessage(tx, role, clusterId, nodeId, messageType, level, subject, body, paramsJSON)
 	if err != nil {
 		return err
 	}
 
+	// TODO 目前只支持边缘节点发送消息，将来要支持NS节点
+	if role == nodeconfigs.NodeRoleNode {
 		// 发送给媒介接收人 - 集群
 		err = SharedMessageTaskDAO.CreateMessageTasks(tx, MessageTaskTarget{
 			ClusterId: clusterId,
@@ -153,6 +159,7 @@ func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, clusterId int64, nodeId in
 				return err
 			}
 		}
+	}
 
 	return nil
 }
@@ -179,7 +186,7 @@ func (this *MessageDAO) CreateMessage(tx *dbs.Tx, adminId int64, userId int64, m
 	op.State = MessageStateEnabled
 	op.IsRead = false
 	op.Day = timeutil.Format("Ymd")
-	op.Hash = this.calHash(subject, body, paramsJSON)
+	op.Hash = this.calHash(nodeconfigs.NodeRoleAdmin, 0, 0, subject, body, paramsJSON)
 	err := this.Save(tx, op)
 	if err != nil {
 		return err
@@ -287,13 +294,14 @@ func (this *MessageDAO) CheckMessageUser(tx *dbs.Tx, messageId int64, adminId in
 }
 
 // 创建消息
-func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) (int64, error) {
+func (this *MessageDAO) createMessage(tx *dbs.Tx, role string, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) (int64, error) {
 	// TODO 检查同样的消息最近是否发送过
 
 	// 创建新消息
 	op := NewMessageOperator()
 	op.AdminId = 0 // TODO
 	op.UserId = 0  // TODO
+	op.Role = role
 	op.ClusterId = clusterId
 	op.NodeId = nodeId
 	op.Type = messageType
@@ -314,7 +322,7 @@ func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64,
 	op.State = MessageStateEnabled
 	op.CreatedAt = time.Now().Unix()
 	op.Day = timeutil.Format("Ymd")
-	op.Hash = this.calHash(subject, body, paramsJSON)
+	op.Hash = this.calHash(role, clusterId, nodeId, subject, body, paramsJSON)
 
 	err := this.Save(tx, op)
 	if err != nil {
@@ -324,10 +332,11 @@ func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64,
 }
 
 // 计算Hash
-func (this *MessageDAO) calHash(subject string, body string, paramsJSON []byte) string {
+func (this *MessageDAO) calHash(role string, clusterId int64, nodeId int64, subject string, body string, paramsJSON []byte) string {
 	h := md5.New()
-	h.Write([]byte(subject))
+	h.Write([]byte(role + "@" + types.String(clusterId) + "@" + types.String(nodeId)))
-	h.Write([]byte(body))
+	h.Write([]byte(subject + "@"))
+	h.Write([]byte(body + "@"))
 	h.Write(paramsJSON)
 	return fmt.Sprintf("%x", h.Sum(nil))
 }

internal/db/models/message_model.go

@@ -5,6 +5,7 @@ type Message struct {
 	Id        uint64 `field:"id"`        // ID
 	AdminId   uint32 `field:"adminId"`   // 管理员ID
 	UserId    uint32 `field:"userId"`    // 用户ID
+	Role      string `field:"role"`      // 角色
 	ClusterId uint32 `field:"clusterId"` // 集群ID
 	NodeId    uint32 `field:"nodeId"`    // 节点ID
 	Level     string `field:"level"`     // 级别
@@ -23,6 +24,7 @@ type MessageOperator struct {
 	Id        interface{} // ID
 	AdminId   interface{} // 管理员ID
 	UserId    interface{} // 用户ID
+	Role      interface{} // 角色
 	ClusterId interface{} // 集群ID
 	NodeId    interface{} // 节点ID
 	Level     interface{} // 级别

internal/db/models/nameservers/ns_node_dao.go

@@ -134,9 +134,9 @@ func (this *NSNodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx, clusterId int64, in
 	case configutils.BoolStateAll:
 		// 所有
 	case configutils.BoolStateYes:
-		query.Where("JSON_EXTRACT(status, '$.isActive') AND UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')<=60")
+		query.Where("(isActive=1 AND JSON_EXTRACT(status, '$.isActive') AND UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')<=60)")
 	case configutils.BoolStateNo:
-		query.Where("(status IS NULL OR NOT JSON_EXTRACT(status, '$.isActive') OR UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')>60)")
+		query.Where("(isActive=0 OR status IS NULL OR NOT JSON_EXTRACT(status, '$.isActive') OR UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')>60)")
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
@@ -167,9 +167,9 @@ func (this *NSNodeDAO) ListAllEnabledNodesMatch(tx *dbs.Tx, clusterId int64, ins
 	case configutils.BoolStateAll:
 		// 所有
 	case configutils.BoolStateYes:
-		query.Where("JSON_EXTRACT(status, '$.isActive') AND UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')<=60")
+		query.Where("(isActive=1 AND JSON_EXTRACT(status, '$.isActive') AND UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')<=60)")
 	case configutils.BoolStateNo:
-		query.Where("(status IS NULL OR NOT JSON_EXTRACT(status, '$.isActive') OR UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')>60)")
+		query.Where("(isActive=0 OR status IS NULL OR NOT JSON_EXTRACT(status, '$.isActive') OR UNIX_TIMESTAMP()-JSON_EXTRACT(status, '$.updatedAt')>60)")
 	}
 
 	if clusterId > 0 {
@@ -428,6 +428,76 @@ func (this *NSNodeDAO) FindNodeClusterId(tx *dbs.Tx, nodeId int64) (int64, error
 		FindInt64Col(0)
 }
 
+// FindNodeActive 检查节点活跃状态
+func (this *NSNodeDAO) FindNodeActive(tx *dbs.Tx, nodeId int64) (bool, error) {
+	isActive, err := this.Query(tx).
+		Pk(nodeId).
+		Result("isActive").
+		FindIntCol(0)
+	if err != nil {
+		return false, err
+	}
+	return isActive == 1, nil
+}
+
+// UpdateNodeActive 修改节点活跃状态
+func (this *NSNodeDAO) UpdateNodeActive(tx *dbs.Tx, nodeId int64, isActive bool) error {
+	if nodeId <= 0 {
+		return errors.New("invalid nodeId")
+	}
+	_, err := this.Query(tx).
+		Pk(nodeId).
+		Set("isActive", isActive).
+		Set("statusIsNotified", false).
+		Update()
+	return err
+}
+
+// UpdateNodeConnectedAPINodes 修改当前连接的API节点
+func (this *NSNodeDAO) UpdateNodeConnectedAPINodes(tx *dbs.Tx, nodeId int64, apiNodeIds []int64) error {
+	if nodeId <= 0 {
+		return errors.New("invalid nodeId")
+	}
+
+	op := NewNSNodeOperator()
+	op.Id = nodeId
+
+	if len(apiNodeIds) > 0 {
+		apiNodeIdsJSON, err := json.Marshal(apiNodeIds)
+		if err != nil {
+			return errors.Wrap(err)
+		}
+		op.ConnectedAPINodes = apiNodeIdsJSON
+	} else {
+		op.ConnectedAPINodes = "[]"
+	}
+	err := this.Save(tx, op)
+	return err
+}
+
+// FindAllNotifyingInactiveNodesWithClusterId 取得某个集群所有等待通知离线离线的节点
+func (this *NSNodeDAO) FindAllNotifyingInactiveNodesWithClusterId(tx *dbs.Tx, clusterId int64) (result []*NSNode, err error) {
+	_, err = this.Query(tx).
+		State(NSNodeStateEnabled).
+		Attr("clusterId", clusterId).
+		Attr("isOn", true).        // 只监控启用的节点
+		Attr("isInstalled", true). // 只监控已经安装的节点
+		Attr("isActive", false).   // 当前已经离线的
+		Attr("statusIsNotified", false).
+		Result("id", "name").
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// UpdateNodeStatusIsNotified 设置状态已经通知
+func (this *NSNodeDAO) UpdateNodeStatusIsNotified(tx *dbs.Tx, nodeId int64) error {
+	return this.Query(tx).
+		Pk(nodeId).
+		Set("statusIsNotified", true).
+		UpdateQuickly()
+}
+
 // NotifyUpdate 通知更新
 func (this *NSNodeDAO) NotifyUpdate(tx *dbs.Tx, nodeId int64) error {
 	// TODO 先什么都不做

internal/db/models/nameservers/ns_node_model.go

@@ -15,6 +15,9 @@ type NSNode struct {
 	InstallStatus     string `field:"installStatus"`     // 安装状态
 	InstallDir        string `field:"installDir"`        // 安装目录
 	State             uint8  `field:"state"`             // 状态
+	IsActive          uint8  `field:"isActive"`          // 是否活跃
+	StatusIsNotified  uint8  `field:"statusIsNotified"`  // 活跃状态已经通知
+	ConnectedAPINodes string `field:"connectedAPINodes"` // 当前连接的API节点
 }
 
 type NSNodeOperator struct {
@@ -31,6 +34,9 @@ type NSNodeOperator struct {
 	InstallStatus     interface{} // 安装状态
 	InstallDir        interface{} // 安装目录
 	State             interface{} // 状态
+	IsActive          interface{} // 是否活跃
+	StatusIsNotified  interface{} // 活跃状态已经通知
+	ConnectedAPINodes interface{} // 当前连接的API节点
 }
 
 func NewNSNodeOperator() *NSNodeOperator {

internal/db/models/node_threshold_dao.go

@@ -252,7 +252,7 @@ func (this *NodeThresholdDAO) FireNodeThreshold(tx *dbs.Tx, role string, nodeId
 					body = strings.Replace(body, "${item.name}", itemName, -1)
 					body = strings.Replace(body, "${value}", fmt.Sprintf("%.2f", paramValue), -1)
 				}
-				err = SharedMessageDAO.CreateNodeMessage(tx, clusterId, nodeId, MessageTypeThresholdSatisfied, MessageLevelWarning, subject, body, maps.Map{}.AsJSON())
+				err = SharedMessageDAO.CreateNodeMessage(tx, role, clusterId, nodeId, MessageTypeThresholdSatisfied, MessageLevelWarning, subject, body, maps.Map{}.AsJSON())
 				if err != nil {
 					return err
 				}

internal/rpc/services/nameservers/service_ns_node.go

@@ -119,6 +119,7 @@ func (this *NSNodeService) ListEnabledNSNodesMatch(ctx context.Context, req *pb.
 			IsOn:          node.IsOn == 1,
 			UniqueId:      node.UniqueId,
 			Secret:        node.Secret,
+			IsActive:      node.IsActive == 1,
 			IsInstalled:   node.IsInstalled == 1,
 			InstallDir:    node.InstallDir,
 			IsUp:          node.IsUp == 1,
@@ -420,3 +421,21 @@ func (this *NSNodeService) DownloadNSNodeInstallationFile(ctx context.Context, r
 		Filename:  filepath.Base(file.Path),
 	}, nil
 }
+
+// UpdateNSNodeConnectedAPINodes 更改节点连接的API节点信息
+func (this *NSNodeService) UpdateNSNodeConnectedAPINodes(ctx context.Context, req *pb.UpdateNSNodeConnectedAPINodesRequest) (*pb.RPCSuccess, error) {
+	// 校验节点
+	_, _, nodeId, err := rpcutils.ValidateRequest(ctx, rpcutils.UserTypeDNS)
+	if err != nil {
+		return nil, err
+	}
+
+	tx := this.NullTx()
+
+	err = nameservers.SharedNSNodeDAO.UpdateNodeConnectedAPINodes(tx, nodeId, req.ApiNodeIds)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+
+	return this.Success()
+}

internal/rpc/services/nameservers/service_ns_node_stream.go

@@ -0,0 +1,290 @@
+package nameservers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/TeaOSLab/EdgeAPI/internal/configs"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/iwind/TeaGo/logs"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// CommandRequest 命令请求相关
+type CommandRequest struct {
+	Id          int64
+	Code        string
+	CommandJSON []byte
+}
+
+type CommandRequestWaiting struct {
+	Timestamp int64
+	Chan      chan *pb.NSNodeStreamMessage
+}
+
+func (this *CommandRequestWaiting) Close() {
+	defer func() {
+		recover()
+	}()
+
+	close(this.Chan)
+}
+
+var responseChanMap = map[int64]*CommandRequestWaiting{} // request id => response
+var commandRequestId = int64(0)
+
+var nodeLocker = &sync.Mutex{}
+var requestChanMap = map[int64]chan *CommandRequest{} // node id => chan
+
+func NextCommandRequestId() int64 {
+	return atomic.AddInt64(&commandRequestId, 1)
+}
+
+func init() {
+	// 清理WaitingChannelMap
+	ticker := time.NewTicker(30 * time.Second)
+	go func() {
+		for range ticker.C {
+			nodeLocker.Lock()
+			for requestId, request := range responseChanMap {
+				if time.Now().Unix()-request.Timestamp > 3600 {
+					responseChanMap[requestId].Close()
+					delete(responseChanMap, requestId)
+				}
+			}
+			nodeLocker.Unlock()
+		}
+	}()
+}
+
+// NsNodeStream 节点stream
+func (this *NSNodeService) NsNodeStream(server pb.NSNodeService_NsNodeStreamServer) error {
+	// TODO 使用此stream快速通知NS节点更新
+	// 校验节点
+	_, _, nodeId, err := rpcutils.ValidateRequest(server.Context(), rpcutils.UserTypeDNS)
+	if err != nil {
+		return err
+	}
+
+	// 返回连接成功
+	{
+		apiConfig, err := configs.SharedAPIConfig()
+		if err != nil {
+			return err
+		}
+		connectedMessage := &messageconfigs.NSConnectedAPINodeMessage{APINodeId: apiConfig.NumberId()}
+		connectedMessageJSON, err := json.Marshal(connectedMessage)
+		if err != nil {
+			return errors.Wrap(err)
+		}
+		err = server.Send(&pb.NSNodeStreamMessage{
+			Code:     messageconfigs.NSMessageCodeConnectedAPINode,
+			DataJSON: connectedMessageJSON,
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	//logs.Println("[RPC]accepted ns node '" + types.String(nodeId) + "' connection")
+
+	tx := this.NullTx()
+
+	// 标记为活跃状态
+	oldIsActive, err := nameservers.SharedNSNodeDAO.FindNodeActive(tx, nodeId)
+	if err != nil {
+		return err
+	}
+	if !oldIsActive {
+		err = nameservers.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, true)
+		if err != nil {
+			return err
+		}
+
+		// 发送恢复消息
+		clusterId, err := nameservers.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
+		if err != nil {
+			return err
+		}
+		nodeName, err := nameservers.SharedNSNodeDAO.FindEnabledNSNodeName(tx, nodeId)
+		if err != nil {
+			return err
+		}
+		subject := "DNS节点\"" + nodeName + "\"已经恢复在线"
+		msg := "DNS节点\"" + nodeName + "\"已经恢复在线"
+		err = models.SharedMessageDAO.CreateNodeMessage(tx, nodeconfigs.NodeRoleDNS, clusterId, nodeId, models.MessageTypeNSNodeActive, models.MessageLevelSuccess, subject, msg, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	nodeLocker.Lock()
+	requestChan, ok := requestChanMap[nodeId]
+	if !ok {
+		requestChan = make(chan *CommandRequest, 1024)
+		requestChanMap[nodeId] = requestChan
+	}
+	nodeLocker.Unlock()
+
+	// 发送请求
+	go func() {
+		for {
+			select {
+			case <-server.Context().Done():
+				return
+			case commandRequest := <-requestChan:
+				// logs.Println("[RPC]sending command '" + commandRequest.Code + "' to node '" + strconv.FormatInt(nodeId, 10) + "'")
+				retries := 3 // 错误重试次数
+				for i := 0; i < retries; i++ {
+					err := server.Send(&pb.NSNodeStreamMessage{
+						RequestId: commandRequest.Id,
+						Code:      commandRequest.Code,
+						DataJSON:  commandRequest.CommandJSON,
+					})
+					if err != nil {
+						if i == retries-1 {
+							logs.Println("[RPC]send command '" + commandRequest.Code + "' failed: " + err.Error())
+						} else {
+							time.Sleep(1 * time.Second)
+						}
+					} else {
+						break
+					}
+				}
+			}
+		}
+	}()
+
+	// 接受请求
+	for {
+		req, err := server.Recv()
+		if err != nil {
+			// 修改节点状态
+			err1 := nameservers.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, false)
+			if err1 != nil {
+				logs.Println(err1.Error())
+			}
+
+			return err
+		}
+
+		func(req *pb.NSNodeStreamMessage) {
+			// 因为 responseChan.Chan 有被关闭的风险，所以我们使用recover防止panic
+			defer func() {
+				recover()
+			}()
+
+			nodeLocker.Lock()
+			responseChan, ok := responseChanMap[req.RequestId]
+			if ok {
+				select {
+				case responseChan.Chan <- req:
+				default:
+
+				}
+			}
+			nodeLocker.Unlock()
+		}(req)
+	}
+}
+
+// SendCommandToNSNode 向节点发送命令
+func (this *NSNodeService) SendCommandToNSNode(ctx context.Context, req *pb.NSNodeStreamMessage) (*pb.NSNodeStreamMessage, error) {
+	// 校验请求
+	_, _, err := this.ValidateAdminAndUser(ctx, 0, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeId := req.NsNodeId
+	if nodeId <= 0 {
+		return nil, errors.New("node id should not be less than 0")
+	}
+
+	nodeLocker.Lock()
+	requestChan, ok := requestChanMap[nodeId]
+	nodeLocker.Unlock()
+
+	if !ok {
+		return &pb.NSNodeStreamMessage{
+			RequestId: req.RequestId,
+			IsOk:      false,
+			Message:   "node '" + strconv.FormatInt(nodeId, 10) + "' not connected yet",
+		}, nil
+	}
+
+	req.RequestId = NextCommandRequestId()
+
+	select {
+	case requestChan <- &CommandRequest{
+		Id:          req.RequestId,
+		Code:        req.Code,
+		CommandJSON: req.DataJSON,
+	}:
+		// 加入到等待队列中
+		respChan := make(chan *pb.NSNodeStreamMessage, 1)
+		waiting := &CommandRequestWaiting{
+			Timestamp: time.Now().Unix(),
+			Chan:      respChan,
+		}
+
+		nodeLocker.Lock()
+		responseChanMap[req.RequestId] = waiting
+		nodeLocker.Unlock()
+
+		// 等待响应
+		timeoutSeconds := req.TimeoutSeconds
+		if timeoutSeconds <= 0 {
+			timeoutSeconds = 10
+		}
+		timeout := time.NewTimer(time.Duration(timeoutSeconds) * time.Second)
+		select {
+		case resp := <-respChan:
+			// 从队列中删除
+			nodeLocker.Lock()
+			delete(responseChanMap, req.RequestId)
+			waiting.Close()
+			nodeLocker.Unlock()
+
+			if resp == nil {
+				return &pb.NSNodeStreamMessage{
+					RequestId: req.RequestId,
+					Code:      req.Code,
+					Message:   "response timeout",
+					IsOk:      false,
+				}, nil
+			}
+
+			return resp, nil
+		case <-timeout.C:
+			// 从队列中删除
+			nodeLocker.Lock()
+			delete(responseChanMap, req.RequestId)
+			waiting.Close()
+			nodeLocker.Unlock()
+
+			return &pb.NSNodeStreamMessage{
+				RequestId: req.RequestId,
+				Code:      req.Code,
+				Message:   "response timeout over " + fmt.Sprintf("%d", timeoutSeconds) + " seconds",
+				IsOk:      false,
+			}, nil
+		}
+	default:
+		return &pb.NSNodeStreamMessage{
+			RequestId: req.RequestId,
+			Code:      req.Code,
+			Message:   "command queue is full over " + strconv.Itoa(len(requestChan)),
+			IsOk:      false,
+		}, nil
+	}
+}

internal/rpc/services/service_message.go

@@ -3,15 +3,17 @@ package services
 import (
 	"context"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 )
 
-// 消息相关服务
+// MessageService 消息相关服务
 type MessageService struct {
 	BaseService
 }
 
-// 计算未读消息数
+// CountUnreadMessages 计算未读消息数
 func (this *MessageService) CountUnreadMessages(ctx context.Context, req *pb.CountUnreadMessagesRequest) (*pb.RPCCountResponse, error) {
 	// 校验请求
 	adminId, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
@@ -28,7 +30,7 @@ func (this *MessageService) CountUnreadMessages(ctx context.Context, req *pb.Cou
 	return this.SuccessCount(count)
 }
 
-// 列出单页未读消息
+// ListUnreadMessages 列出单页未读消息
 func (this *MessageService) ListUnreadMessages(ctx context.Context, req *pb.ListUnreadMessagesRequest) (*pb.ListUnreadMessagesResponse, error) {
 	// 校验请求
 	adminId, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
@@ -48,6 +50,8 @@ func (this *MessageService) ListUnreadMessages(ctx context.Context, req *pb.List
 		var pbNode *pb.Node = nil
 
 		if message.ClusterId > 0 {
+			switch message.Role {
+			case nodeconfigs.NodeRoleNode:
 				cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, int64(message.ClusterId))
 				if err != nil {
 					return nil, err
@@ -58,9 +62,23 @@ func (this *MessageService) ListUnreadMessages(ctx context.Context, req *pb.List
 						Name: cluster.Name,
 					}
 				}
+			case nodeconfigs.NodeRoleDNS:
+				cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(message.ClusterId))
+				if err != nil {
+					return nil, err
+				}
+				if cluster != nil {
+					pbCluster = &pb.NodeCluster{
+						Id:   int64(cluster.Id),
+						Name: cluster.Name,
+					}
+				}
+			}
 		}
 
 		if message.NodeId > 0 {
+			switch message.Role {
+			case nodeconfigs.NodeRoleNode:
 				node, err := models.SharedNodeDAO.FindEnabledNode(tx, int64(message.NodeId))
 				if err != nil {
 					return nil, err
@@ -71,10 +89,23 @@ func (this *MessageService) ListUnreadMessages(ctx context.Context, req *pb.List
 						Name: node.Name,
 					}
 				}
+			case nodeconfigs.NodeRoleDNS:
+				node, err := nameservers.SharedNSNodeDAO.FindEnabledNSNode(tx, int64(message.NodeId))
+				if err != nil {
+					return nil, err
+				}
+				if node != nil {
+					pbNode = &pb.Node{
+						Id:   int64(node.Id),
+						Name: node.Name,
+					}
+				}
+			}
 		}
 
 		result = append(result, &pb.Message{
 			Id:          int64(message.Id),
+			Role:        message.Role,
 			Type:        message.Type,
 			Body:        message.Body,
 			Level:       message.Level,
@@ -89,7 +120,7 @@ func (this *MessageService) ListUnreadMessages(ctx context.Context, req *pb.List
 	return &pb.ListUnreadMessagesResponse{Messages: result}, nil
 }
 
-// 设置消息已读状态
+// UpdateMessageRead 设置消息已读状态
 func (this *MessageService) UpdateMessageRead(ctx context.Context, req *pb.UpdateMessageReadRequest) (*pb.RPCSuccess, error) {
 	// 校验请求
 	adminId, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
@@ -115,7 +146,7 @@ func (this *MessageService) UpdateMessageRead(ctx context.Context, req *pb.Updat
 	return this.Success()
 }
 
-// 设置一组消息已读状态
+// UpdateMessagesRead 设置一组消息已读状态
 func (this *MessageService) UpdateMessagesRead(ctx context.Context, req *pb.UpdateMessagesReadRequest) (*pb.RPCSuccess, error) {
 	// 校验请求
 	adminId, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
@@ -143,7 +174,7 @@ func (this *MessageService) UpdateMessagesRead(ctx context.Context, req *pb.Upda
 	return this.Success()
 }
 
-// 设置所有消息为已读
+// UpdateAllMessagesRead 设置所有消息为已读
 func (this *MessageService) UpdateAllMessagesRead(ctx context.Context, req *pb.UpdateAllMessagesReadRequest) (*pb.RPCSuccess, error) {
 	// 校验请求
 	// 校验请求

internal/rpc/services/service_node_stream.go

@@ -9,6 +9,7 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/iwind/TeaGo/logs"
 	"strconv"
@@ -119,7 +120,7 @@ func (this *NodeService) NodeStream(server pb.NodeService_NodeStreamServer) erro
 		}
 		subject := "节点\"" + nodeName + "\"已经恢复在线"
 		msg := "节点\"" + nodeName + "\"已经恢复在线"
-		err = models.SharedMessageDAO.CreateNodeMessage(tx, clusterId, nodeId, models.MessageTypeNodeActive, models.MessageLevelSuccess, subject, msg, nil)
+		err = models.SharedMessageDAO.CreateNodeMessage(tx, nodeconfigs.NodeRoleNode, clusterId, nodeId, models.MessageTypeNodeActive, models.MessageLevelSuccess, subject, msg, nil)
 		if err != nil {
 			return err
 		}

internal/tasks/health_check_cluster_task.go

@@ -6,6 +6,7 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
 	"github.com/iwind/TeaGo/logs"
@@ -132,7 +133,7 @@ func (this *HealthCheckClusterTask) loop(seconds int64) error {
 			return err
 		}
 		message := "有" + numberutils.FormatInt(len(failedResults)) + "个节点在健康检查中出现问题"
-		err = models.NewMessageDAO().CreateClusterMessage(nil, this.clusterId, models.MessageTypeHealthCheckFailed, models.MessageLevelError, message, message, failedResultsJSON)
+		err = models.NewMessageDAO().CreateClusterMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, models.MessageTypeHealthCheckFailed, models.MessageLevelError, message, message, failedResultsJSON)
 		if err != nil {
 			return err
 		}

internal/tasks/health_check_executor.go

@@ -139,10 +139,10 @@ func (this *HealthCheckExecutor) Run() ([]*HealthCheckResult, error) {
 							// 通知恢复或下线
 							if result.IsOk {
 								message := "健康检查成功，节点\"" + result.Node.Name + "\"已恢复上线"
-								err = models.NewMessageDAO().CreateNodeMessage(nil, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeUp, models.MessageLevelSuccess, message, message, nil)
+								err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeUp, models.MessageLevelSuccess, message, message, nil)
 							} else {
 								message := "健康检查失败，节点\"" + result.Node.Name + "\"已自动下线"
-								err = models.NewMessageDAO().CreateNodeMessage(nil, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeDown, models.MessageLevelError, message, message, nil)
+								err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeDown, models.MessageLevelError, message, message, nil)
 							}
 						}
 					}

internal/tasks/node_monitor_task.go

@@ -3,6 +3,7 @@ package tasks
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/logs"
@@ -24,7 +25,7 @@ func init() {
 	})
 }
 
-// NodeMonitorTask 健康节点任务
+// NodeMonitorTask 边缘节点监控任务
 type NodeMonitorTask struct {
 	intervalSeconds int
 }
@@ -82,7 +83,7 @@ func (this *NodeMonitorTask) monitorCluster(cluster *models.NodeCluster) error {
 	for _, node := range inactiveNodes {
 		subject := "节点\"" + node.Name + "\"已处于离线状态"
 		msg := "节点\"" + node.Name + "\"已处于离线状态"
-		err = models.SharedMessageDAO.CreateNodeMessage(nil, clusterId, int64(node.Id), models.MessageTypeNodeInactive, models.LevelError, subject, msg, nil)
+		err = models.SharedMessageDAO.CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, clusterId, int64(node.Id), models.MessageTypeNodeInactive, models.LevelError, subject, msg, nil)
 		if err != nil {
 			return err
 		}

internal/tasks/ns_node_monitor_task.go

@@ -0,0 +1,107 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
+	"time"
+)
+
+func init() {
+	dbs.OnReady(func() {
+		task := NewNSNodeMonitorTask(60)
+		ticker := time.NewTicker(60 * time.Second)
+		go func() {
+			for range ticker.C {
+				err := task.loop()
+				if err != nil {
+					logs.Println("[TASK][NS_NODE_MONITOR]" + err.Error())
+				}
+			}
+		}()
+	})
+}
+
+// NSNodeMonitorTask 边缘节点监控任务
+type NSNodeMonitorTask struct {
+	intervalSeconds int
+}
+
+func NewNSNodeMonitorTask(intervalSeconds int) *NSNodeMonitorTask {
+	return &NSNodeMonitorTask{
+		intervalSeconds: intervalSeconds,
+	}
+}
+
+func (this *NSNodeMonitorTask) Run() {
+
+}
+
+func (this *NSNodeMonitorTask) loop() error {
+	// 检查上次运行时间，防止重复运行
+	settingKey := systemconfigs.SettingCodeNSNodeMonitor + "Loop"
+	timestamp := time.Now().Unix()
+	c, err := models.SharedSysSettingDAO.CompareInt64Setting(nil, settingKey, timestamp-int64(this.intervalSeconds))
+	if err != nil {
+		return err
+	}
+	if c > 0 {
+		return nil
+	}
+
+	// 记录时间
+	err = models.SharedSysSettingDAO.UpdateSetting(nil, settingKey, []byte(numberutils.FormatInt64(timestamp)))
+	if err != nil {
+		return err
+	}
+
+	clusters, err := nameservers.SharedNSClusterDAO.FindAllEnabledClusters(nil)
+	if err != nil {
+		return err
+	}
+	for _, cluster := range clusters {
+		err := this.monitorCluster(cluster)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *NSNodeMonitorTask) monitorCluster(cluster *nameservers.NSCluster) error {
+	clusterId := int64(cluster.Id)
+
+	// 检查离线节点
+	inactiveNodes, err := nameservers.SharedNSNodeDAO.FindAllNotifyingInactiveNodesWithClusterId(nil, clusterId)
+	if err != nil {
+		return err
+	}
+	for _, node := range inactiveNodes {
+		subject := "DNS节点\"" + node.Name + "\"已处于离线状态"
+		msg := "DNS节点\"" + node.Name + "\"已处于离线状态"
+		err = models.SharedMessageDAO.CreateNodeMessage(nil, nodeconfigs.NodeRoleDNS, clusterId, int64(node.Id), models.MessageTypeNSNodeInactive, models.LevelError, subject, msg, nil)
+		if err != nil {
+			return err
+		}
+
+		// 修改在线状态
+		err = nameservers.SharedNSNodeDAO.UpdateNodeStatusIsNotified(nil, int64(node.Id))
+		if err != nil {
+			return err
+		}
+	}
+
+	// TODO 检查恢复连接
+
+	// 检查CPU、内存、磁盘不足节点，而且离线的节点不再重复提示
+	// TODO 需要实现
+
+	// TODO 检查53/tcp、53/udp是否能够访问
+
+	return nil
+}