EdgeAPI/internal/tasks/health_check_cluster_task.go

package tasks

import (
	"bytes"
	"encoding/json"
	"strings"
	"time"

	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
	"github.com/TeaOSLab/EdgeAPI/internal/goman"
	"github.com/TeaOSLab/EdgeAPI/internal/utils"
	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
	"github.com/iwind/TeaGo/maps"
	"github.com/iwind/TeaGo/types"
)

// HealthCheckClusterTask 单个集群的健康检查任务
type HealthCheckClusterTask struct {
	BaseTask

	clusterId int64
	config    *serverconfigs.HealthCheckConfig
	ticker    *utils.Ticker

	notifiedTime time.Time
}

// NewHealthCheckClusterTask 创建新任务
func NewHealthCheckClusterTask(clusterId int64, config *serverconfigs.HealthCheckConfig) *HealthCheckClusterTask {
	return &HealthCheckClusterTask{
		clusterId: clusterId,
		config:    config,
	}
}

// Reset 重置配置
func (this *HealthCheckClusterTask) Reset(config *serverconfigs.HealthCheckConfig) {
	// 检查是否有变化
	oldJSON, err := json.Marshal(this.config)
	if err != nil {
		this.logErr("HealthCheckClusterTask", err.Error())
		return
	}
	newJSON, err := json.Marshal(config)
	if err != nil {
		this.logErr("HealthCheckClusterTask", err.Error())
		return
	}
	if !bytes.Equal(oldJSON, newJSON) {
		this.config = config
		this.Run()
	}
}

// Run 执行
func (this *HealthCheckClusterTask) Run() {
	this.Stop()

	if this.config == nil {
		return
	}
	if !this.config.IsOn {
		return
	}
	if this.config.Interval == nil {
		return
	}
	var duration = this.config.Interval.Duration()
	if duration <= 0 {
		return
	}
	var ticker = utils.NewTicker(duration)
	goman.New(func() {
		for ticker.Wait() {
			err := this.Loop()
			if err != nil {
				this.logErr("HealthCheckClusterTask", err.Error())
			}
		}
	})
	this.ticker = ticker
}

// Stop 停止
func (this *HealthCheckClusterTask) Stop() {
	if this.ticker == nil {
		return
	}
	this.ticker.Stop()
	this.ticker = nil
}

// Loop 单个循环任务
func (this *HealthCheckClusterTask) Loop() error {
	// 检查是否为主节点
	if !this.IsPrimaryNode() {
		return nil
	}

	// 开始运行
	var executor = NewHealthCheckExecutor(this.clusterId)
	results, err := executor.Run()
	if err != nil {
		return err
	}

	var failedResults = []maps.Map{}
	for _, result := range results {
		if !result.IsOk {
			failedResults = append(failedResults, maps.Map{
				"node": maps.Map{
					"id":   result.Node.Id,
					"name": result.Node.Name,
				},
				"isOk":     false,
				"error":    result.Error,
				"nodeAddr": result.NodeAddr,
			})
		}
	}

	if len(failedResults) > 0 {
		// 10分钟内不重复提醒
		if time.Since(this.notifiedTime) > 10*time.Minute {
			this.notifiedTime = time.Now()

			failedResultsJSON, err := json.Marshal(failedResults)
			if err != nil {
				return err
			}
			var subject = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题"
			var message = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题："
			var failedDescriptions = []string{}
			var failedIndex int
			for _, result := range results {
				if result.IsOk || result.Node == nil {
					continue
				}
				failedIndex++
				failedDescriptions = append(failedDescriptions, "节点"+types.String(failedIndex)+"："+result.Node.Name+"，IP："+result.NodeAddr)
			}

			const maxNodeDescriptions = 10
			var isOverMax = false
			if len(failedDescriptions) > maxNodeDescriptions {
				failedDescriptions = failedDescriptions[:maxNodeDescriptions]
				isOverMax = true
			}
			message += strings.Join(failedDescriptions, "；")
			if isOverMax {
				message += " ..."
			} else {
				message += "。"
			}

			err = models.NewMessageDAO().CreateClusterMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, models.MessageTypeHealthCheckFailed, models.MessageLevelError, subject, subject, message, failedResultsJSON)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Config 获取当前配置
func (this *HealthCheckClusterTask) Config() *serverconfigs.HealthCheckConfig {
	return this.config
}