实现实时展示访问日志

internal/db/models/http_access_log_dao.go

@@ -2,13 +2,19 @@ package models
 
 import (
 	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/configs"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/logs"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -28,23 +34,28 @@ func NewHTTPAccessLogDAO() *HTTPAccessLogDAO {
 }
 
 // 创建访问日志
-func CreateHTTPAccessLogs(accessLogs []*pb.HTTPAccessLog) error {
+func (this *HTTPAccessLogDAO) CreateHTTPAccessLogs(accessLogs []*pb.HTTPAccessLog) error {
 	dao := randomAccessLogDAO()
 	if dao == nil {
-		dao = SharedHTTPAccessLogDAO
+		dao = &HTTPAccessLogDAOWrapper{
+			DAO:    SharedHTTPAccessLogDAO,
+			NodeId: 0,
+		}
 	}
-	return CreateHTTPAccessLogsWithDAO(dao, accessLogs)
+	return this.CreateHTTPAccessLogsWithDAO(dao, accessLogs)
 }
 
 // 使用特定的DAO创建访问日志
-func CreateHTTPAccessLogsWithDAO(dao *HTTPAccessLogDAO, accessLogs []*pb.HTTPAccessLog) error {
-	if dao == nil {
+func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccessLogDAOWrapper, accessLogs []*pb.HTTPAccessLog) error {
+	if daoWrapper == nil {
 		return errors.New("dao should not be nil")
 	}
 	if len(accessLogs) == 0 {
 		return nil
 	}
 
+	dao := daoWrapper.DAO
+
 	// TODO 改成事务批量提交，以加快速度
 
 	for _, accessLog := range accessLogs {
@@ -59,7 +70,7 @@ func CreateHTTPAccessLogsWithDAO(dao *HTTPAccessLogDAO, accessLogs []*pb.HTTPAcc
 		fields["nodeId"] = accessLog.NodeId
 		fields["status"] = accessLog.Status
 		fields["createdAt"] = accessLog.Timestamp
-		fields["day"] = day
+		fields["requestId"] = accessLog.RequestId + strconv.FormatInt(time.Now().UnixNano(), 10) + configs.PaddingId
 
 		content, err := json.Marshal(accessLog)
 		if err != nil {
@@ -91,3 +102,137 @@ func CreateHTTPAccessLogsWithDAO(dao *HTTPAccessLogDAO, accessLogs []*pb.HTTPAcc
 
 	return nil
 }
+
+// 读取往前的 单页访问日志
+func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
+	if len(day) != 8 {
+		return
+	}
+
+	// 限制能查询的最大条数，防止占用内存过多
+	if size > 1000 {
+		size = 1000
+	}
+
+	result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse)
+	if err != nil || int64(len(result)) < size {
+		return
+	}
+
+	moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse)
+	hasMore = len(moreResult) > 0
+	return
+}
+
+// 读取往前的单页访问日志
+func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
+	if size <= 0 {
+		return nil, lastRequestId, nil
+	}
+
+	accessLogLocker.RLock()
+	daoList := []*HTTPAccessLogDAOWrapper{}
+	for _, daoWrapper := range accessLogDAOMapping {
+		daoList = append(daoList, daoWrapper)
+	}
+	accessLogLocker.RUnlock()
+
+	if len(daoList) == 0 {
+		daoList = []*HTTPAccessLogDAOWrapper{{
+			DAO:    SharedHTTPAccessLogDAO,
+			NodeId: 0,
+		}}
+	}
+
+	locker := sync.Mutex{}
+
+	count := len(daoList)
+	wg := &sync.WaitGroup{}
+	wg.Add(count)
+	for _, daoWrapper := range daoList {
+		go func(daoWrapper *HTTPAccessLogDAOWrapper) {
+			defer wg.Done()
+
+			dao := daoWrapper.DAO
+
+			tableName, exists, err := findAccessLogTableName(dao.Instance, day)
+			if !exists {
+				// 表格不存在则跳过
+				return
+			}
+			if err != nil {
+				logs.Println("[DB_NODE]" + err.Error())
+				return
+			}
+
+			query := dao.Query()
+
+			// 条件
+			if serverId > 0 {
+				query.Attr("serverId", serverId)
+			}
+
+			// offset
+			if len(lastRequestId) > 0 {
+				if !reverse {
+					query.Where("requestId<:requestId").
+						Param("requestId", lastRequestId)
+				} else {
+					query.Where("requestId>:requestId").
+						Param("requestId", lastRequestId)
+				}
+			}
+
+			if !reverse {
+				query.Desc("requestId")
+			} else {
+				query.Asc("requestId")
+			}
+
+			// 开始查询
+			ones, err := query.
+				Table(tableName).
+				Limit(size).
+				FindAll()
+			if err != nil {
+				logs.Println("[DB_NODE]" + err.Error())
+				return
+			}
+			locker.Lock()
+			for _, one := range ones {
+				accessLog := one.(*HTTPAccessLog)
+				result = append(result, accessLog)
+			}
+			locker.Unlock()
+		}(daoWrapper)
+	}
+	wg.Wait()
+
+	if len(result) == 0 {
+		return nil, lastRequestId, nil
+	}
+
+	// 按照requestId排序
+	sort.Slice(result, func(i, j int) bool {
+		if !reverse {
+			return result[i].RequestId > result[j].RequestId
+		} else {
+			return result[i].RequestId < result[j].RequestId
+		}
+	})
+
+	if int64(len(result)) > size {
+		result = result[:size]
+	}
+
+	requestId := result[len(result)-1].RequestId
+	if reverse {
+		lists.Reverse(result)
+	}
+
+	if !reverse {
+		return result, requestId, nil
+	} else {
+		return result, requestId, nil
+	}
+}