增加服务带宽统计

internal/db/models/metric_stat_dao.go

@@ -691,7 +691,6 @@ func (this *MetricStatDAO) Clean(tx *dbs.Tx) error {
 						Table(table).
 						Attr("itemId", item.Id).
 						Lte("createdDay", expiresDay).
-						UseIndex("createdDay").
 						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
 						Delete()
 					return err

internal/db/models/metric_sum_stat_dao.go

@@ -237,7 +237,6 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 						Attr("itemId", item.Id).
 						Where("(createdDay IS NULL OR createdDay<:day)").
 						Param("day", expiresDay).
-						UseIndex("createdDay").
 						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
 						Delete()
 					return err

internal/db/models/stats/server_bandwidth_stat_dao.go

@@ -0,0 +1,128 @@
+package stats
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
+	"time"
+)
+
+type ServerBandwidthStatDAO dbs.DAO
+
+const (
+	ServerBandwidthStatTablePartials = 20 // 分表数量
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
+		goman.New(func() {
+			for range ticker.C {
+				err := SharedServerBandwidthStatDAO.Clean(nil)
+				if err != nil {
+					remotelogs.Error("SharedServerBandwidthStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		})
+	})
+}
+
+func NewServerBandwidthStatDAO() *ServerBandwidthStatDAO {
+	return dbs.NewDAO(&ServerBandwidthStatDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeServerBandwidthStats",
+			Model:  new(ServerBandwidthStat),
+			PkName: "id",
+		},
+	}).(*ServerBandwidthStatDAO)
+}
+
+var SharedServerBandwidthStatDAO *ServerBandwidthStatDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedServerBandwidthStatDAO = NewServerBandwidthStatDAO()
+	})
+}
+
+// UpdateServerBandwidth 写入数据
+func (this *ServerBandwidthStatDAO) UpdateServerBandwidth(tx *dbs.Tx, serverId int64, day string, timeAt string, bytes int64) error {
+	if serverId <= 0 {
+		return errors.New("invalid server id '" + types.String(serverId) + "'")
+	}
+
+	return this.Query(tx).
+		Table(this.partialTable(serverId)).
+		Param("bytes", bytes).
+		InsertOrUpdateQuickly(maps.Map{
+			"serverId": serverId,
+			"day":      day,
+			"timeAt":   timeAt,
+			"bytes":    bytes,
+		}, maps.Map{
+			"bytes": dbs.SQL("bytes+:bytes"),
+		})
+}
+
+// FindServerStats 查找某个时间段的带宽统计
+// 参数：
+//   - day YYYYMMDD
+//   - timeAt HHII
+func (this *ServerBandwidthStatDAO) FindServerStats(tx *dbs.Tx, serverId int64, day string, timeFrom string, timeTo string) (result []*ServerBandwidthStat, err error) {
+	_, err = this.Query(tx).
+		Table(this.partialTable(serverId)).
+		Attr("serverId", serverId).
+		Attr("day", day).
+		Between("timeAt", timeFrom, timeTo).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// Clean 清理过期数据
+func (this *ServerBandwidthStatDAO) Clean(tx *dbs.Tx) error {
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -62)) // 保留大约2个月的数据
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Lt("day", day).
+			Delete()
+		return err
+	})
+}
+
+// 批量执行
+func (this *ServerBandwidthStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(ServerBandwidthStatTablePartials)
+	var resultErr error
+	for i := 0; i < ServerBandwidthStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}
+
+// 获取分区表
+func (this *ServerBandwidthStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(ServerBandwidthStatTablePartials))
+}

internal/db/models/stats/server_bandwidth_stat_dao_test.go

@@ -0,0 +1,48 @@
+package stats_test
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/rands"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"testing"
+	"time"
+)
+
+func TestServerBandwidthStatDAO_UpdateServerBandwidth(t *testing.T) {
+	var dao = stats.NewServerBandwidthStatDAO()
+	var tx *dbs.Tx
+	err := dao.UpdateServerBandwidth(tx, 1, timeutil.Format("Ymd"), timeutil.Format("Hi"), 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestSeverBandwidthStatDAO_InsertManyStats(t *testing.T) {
+	var dao = stats.NewServerBandwidthStatDAO()
+	var tx *dbs.Tx
+	for i := 0; i < 1_000_000; i++ {
+		var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -rands.Int(0, 200)))
+		var minute = fmt.Sprintf("%02d%02d", rands.Int(0, 23), rands.Int(0, 59))
+		err := dao.UpdateServerBandwidth(tx, 1, day, minute, 1024)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	t.Log("ok")
+}
+
+func TestServerBandwidthStatDAO_Clean(t *testing.T) {
+	var dao = stats.NewServerBandwidthStatDAO()
+	var tx *dbs.Tx
+	var before = time.Now()
+	err := dao.Clean(tx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok", time.Since(before).Seconds()*1000, "ms")
+}

internal/db/models/stats/server_bandwidth_stat_model.go

@@ -0,0 +1,22 @@
+package stats
+
+// ServerBandwidthStat 服务峰值带宽统计
+type ServerBandwidthStat struct {
+	Id       uint64 `field:"id"`       // ID
+	ServerId uint64 `field:"serverId"` // 服务ID
+	Day      string `field:"day"`      // 日期YYYYMMDD
+	TimeAt   string `field:"timeAt"`   // 时间点HHMM
+	Bytes    uint64 `field:"bytes"`    // 带宽字节
+}
+
+type ServerBandwidthStatOperator struct {
+	Id       interface{} // ID
+	ServerId interface{} // 服务ID
+	Day      interface{} // 日期YYYYMMDD
+	TimeAt   interface{} // 时间点HHMM
+	Bytes    interface{} // 带宽字节
+}
+
+func NewServerBandwidthStatOperator() *ServerBandwidthStatOperator {
+	return &ServerBandwidthStatOperator{}
+}

internal/db/models/stats/server_bandwidth_stat_model_ext.go

@@ -0,0 +1 @@
+package stats