访问日志可以使用分表查询

2025-11-03 20:40:26 +08:00 · 2022-04-17 16:18:43 +08:00
parent 53b1b2c572
commit 9d2eb11e1f
12 changed files with 212 additions and 17 deletions
--- a/internal/web/actions/default/servers/components/waf/log.go
+++ b/internal/web/actions/default/servers/components/waf/log.go
@@ -25,6 +25,7 @@ func (this *LogAction) RunGet(params struct {
 	RequestId        string
 	FirewallPolicyId int64
 	GroupId          int64
+	Partition        int32 `default:"-1"`
 }) {
 	if len(params.Day) == 0 {
 		params.Day = timeutil.Format("Y-m-d")
@@ -42,6 +43,7 @@ func (this *LogAction) RunGet(params struct {
 		size := int64(10)

 		resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition:           params.Partition,
 			RequestId:           params.RequestId,
 			FirewallPolicyId:    params.FirewallPolicyId,
 			FirewallRuleGroupId: params.GroupId,
@@ -74,6 +76,7 @@ func (this *LogAction) RunGet(params struct {
 		if len(params.RequestId) > 0 {
 			this.Data["hasPrev"] = true
 			prevResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+				Partition:           params.Partition,
 				RequestId:           params.RequestId,
 				FirewallPolicyId:    params.FirewallPolicyId,
 				FirewallRuleGroupId: params.GroupId,
--- a/internal/web/actions/default/servers/ipbox/index.go
+++ b/internal/web/actions/default/servers/ipbox/index.go
@@ -20,6 +20,8 @@ func (this *IndexAction) Init() {

 func (this *IndexAction) RunGet(params struct {
 	Ip string
+
+	Partition int32 `default:"-1"`
 }) {
 	this.Data["ip"] = params.Ip

@@ -103,9 +105,10 @@ func (this *IndexAction) RunGet(params struct {

 	// 访问日志
 	accessLogsResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
-		Day:  timeutil.Format("Ymd"),
-		Ip:   params.Ip,
-		Size: 20,
+		Partition: params.Partition,
+		Day:       timeutil.Format("Ymd"),
+		Ip:        params.Ip,
+		Size:      20,
 	})
 	if err != nil {
 		this.ErrorPage(err)
@@ -115,9 +118,10 @@ func (this *IndexAction) RunGet(params struct {
 	if len(accessLogs) == 0 {
 		// 查询昨天
 		accessLogsResp, err = this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
-			Day:  timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)),
-			Ip:   params.Ip,
-			Size: 20,
+			Partition: params.Partition,
+			Day:       timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)),
+			Ip:        params.Ip,
+			Size:      20,
 		})
 		if err != nil {
 			this.ErrorPage(err)
--- a/internal/web/actions/default/servers/iplists/accessLogsPopup.go
+++ b/internal/web/actions/default/servers/iplists/accessLogsPopup.go
@@ -32,20 +32,38 @@ func (this *AccessLogsPopupAction) RunGet(params struct {
 	this.Data["ipFrom"] = item.IpFrom
 	this.Data["ipTo"] = item.IpTo

-	accessLogsResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
-		Day:     timeutil.Format("Ymd"),
-		Keyword: "ip:" + item.IpFrom + "," + item.IpTo,
-		Size:    10,
-	})
+	// 多找几个Partition
+	var day = timeutil.Format("Ymd")
+	partitionsResp, err := this.RPC().HTTPAccessLogRPC().FindHTTPAccessLogPartitions(this.AdminContext(), &pb.FindHTTPAccessLogPartitionsRequest{Day: day})
 	if err != nil {
 		this.ErrorPage(err)
 		return
 	}
-	var accessLogs = accessLogsResp.HttpAccessLogs
-	if len(accessLogs) == 0 {
-		accessLogs = []*pb.HTTPAccessLog{}
+
+	var hasAccessLogs = false
+
+	for _, partition := range partitionsResp.ReversePartitions {
+		accessLogsResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition: partition,
+			Day:       day,
+			Keyword:   "ip:" + item.IpFrom + "," + item.IpTo,
+			Size:      10,
+		})
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+		var accessLogs = accessLogsResp.HttpAccessLogs
+		if len(accessLogs) > 0 {
+			this.Data["accessLogs"] = accessLogs
+			hasAccessLogs = true
+			break
+		}
+	}
+
+	if !hasAccessLogs {
+		this.Data["accessLogs"] = []interface{}{}
 	}
-	this.Data["accessLogs"] = accessLogs

 	this.Show()
 }
--- a/internal/web/actions/default/servers/logs/checkLog.go
+++ b/internal/web/actions/default/servers/logs/checkLog.go
@@ -0,0 +1,55 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+)
+
+// CheckLogAction 检查是否有日志
+type CheckLogAction struct {
+	actionutils.ParentAction
+}
+
+func (this *CheckLogAction) RunPost(params struct {
+	Day       string
+	Partition int32
+
+	ServerId          int64
+	RequestId         string
+	ClusterId         int64
+	NodeId            int64
+	HasError          bool
+	HasFirewallPolicy bool
+	Keyword           string
+	Ip                string
+	Domain            string
+
+	Hour string
+}) {
+	resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+		Partition:         params.Partition,
+		RequestId:         params.RequestId,
+		NodeClusterId:     params.ClusterId,
+		NodeId:            params.NodeId,
+		ServerId:          params.ServerId,
+		HasError:          params.HasError,
+		HasFirewallPolicy: params.HasFirewallPolicy,
+		Day:               params.Day,
+		HourFrom:          params.Hour,
+		HourTo:            params.Hour,
+		Keyword:           params.Keyword,
+		Ip:                params.Ip,
+		Domain:            params.Domain,
+		Size:              1,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	this.Data["hasLogs"] = len(resp.HttpAccessLogs) > 0
+
+	this.Success()
+}
--- a/internal/web/actions/default/servers/logs/index.go
+++ b/internal/web/actions/default/servers/logs/index.go
@@ -31,6 +31,7 @@ func (this *IndexAction) RunGet(params struct {
 	Domain    string
 	HasError  int
 	HasWAF    int
+	Partition int32 `default:"-1"`

 	RequestId string
 	ServerId  int64
@@ -56,6 +57,7 @@ func (this *IndexAction) RunGet(params struct {
 	this.Data["pageSize"] = params.PageSize
 	this.Data["isSlowQuery"] = false
 	this.Data["slowQueryCost"] = ""
+	this.Data["partition"] = params.Partition

 	day := params.Day
 	ipList := []string{}
@@ -71,6 +73,7 @@ func (this *IndexAction) RunGet(params struct {

 		var before = time.Now()
 		resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition:         params.Partition,
 			RequestId:         params.RequestId,
 			NodeClusterId:     params.ClusterId,
 			NodeId:            params.NodeId,
@@ -117,6 +120,7 @@ func (this *IndexAction) RunGet(params struct {
 		if len(params.RequestId) > 0 {
 			this.Data["hasPrev"] = true
 			prevResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+				Partition:         params.Partition,
 				RequestId:         params.RequestId,
 				NodeClusterId:     params.ClusterId,
 				NodeId:            params.NodeId,
--- a/internal/web/actions/default/servers/logs/init.go
+++ b/internal/web/actions/default/servers/logs/init.go
@@ -17,6 +17,7 @@ func init() {
 			Prefix("/servers/logs").
 			Get("", new(IndexAction)).
 			GetPost("/settings", new(SettingsAction)).
+			Post("/partitionData", new(PartitionDataAction)).
 			EndAll()
 	})
 }
--- a/internal/web/actions/default/servers/logs/partitionData.go
+++ b/internal/web/actions/default/servers/logs/partitionData.go
@@ -0,0 +1,37 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"strings"
+)
+
+// PartitionDataAction 读取分区表
+type PartitionDataAction struct {
+	actionutils.ParentAction
+}
+
+func (this *PartitionDataAction) RunPost(params struct {
+	Day string
+}) {
+	var day = params.Day
+	day = strings.ReplaceAll(day, "-", "")
+
+	resp, err := this.RPC().HTTPAccessLogRPC().FindHTTPAccessLogPartitions(this.AdminContext(), &pb.FindHTTPAccessLogPartitionsRequest{
+		Day: day,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	if len(resp.Partitions) > 0 {
+		this.Data["partitions"] = resp.Partitions
+	} else {
+		this.Data["partitions"] = []int32{}
+	}
+
+	this.Success()
+}
--- a/internal/web/actions/default/servers/server/log/history.go
+++ b/internal/web/actions/default/servers/server/log/history.go
@@ -33,6 +33,8 @@ func (this *HistoryAction) RunGet(params struct {
 	ClusterId int64
 	NodeId    int64

+	Partition int32 `default:"-1"`
+
 	PageSize int
 }) {
 	if len(params.Day) == 0 {
@@ -65,6 +67,7 @@ func (this *HistoryAction) RunGet(params struct {
 		this.Data["hasError"] = params.HasError

 		resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition:         params.Partition,
 			RequestId:         params.RequestId,
 			ServerId:          params.ServerId,
 			HasError:          params.HasError > 0,
@@ -105,6 +108,7 @@ func (this *HistoryAction) RunGet(params struct {
 		if len(params.RequestId) > 0 {
 			this.Data["hasPrev"] = true
 			prevResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+				Partition:         params.Partition,
 				RequestId:         params.RequestId,
 				ServerId:          params.ServerId,
 				HasError:          params.HasError > 0,
--- a/internal/web/actions/default/servers/server/log/index.go
+++ b/internal/web/actions/default/servers/server/log/index.go
@@ -57,10 +57,13 @@ func (this *IndexAction) RunPost(params struct {
 	ClusterId int64
 	NodeId    int64

+	Partition int32 `default:"-1"`
+
 	Must *actions.Must
 }) {
 	isReverse := len(params.RequestId) > 0
 	accessLogsResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+		Partition:     params.Partition,
 		ServerId:      params.ServerId,
 		RequestId:     params.RequestId,
 		Size:          20,
--- a/internal/web/actions/default/servers/server/log/today.go
+++ b/internal/web/actions/default/servers/server/log/today.go
@@ -27,6 +27,8 @@ func (this *TodayAction) RunGet(params struct {
 	ClusterId int64
 	NodeId    int64

+	Partition int32 `default:"-1"`
+
 	PageSize int
 }) {
 	this.Data["pageSize"] = params.PageSize
@@ -46,6 +48,7 @@ func (this *TodayAction) RunGet(params struct {
 	this.Data["nodeId"] = params.NodeId

 	resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+		Partition:         params.Partition,
 		RequestId:         params.RequestId,
 		ServerId:          params.ServerId,
 		HasError:          params.HasError > 0,
@@ -85,6 +88,7 @@ func (this *TodayAction) RunGet(params struct {
 	if len(params.RequestId) > 0 {
 		this.Data["hasPrev"] = true
 		prevResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition:         params.Partition,
 			RequestId:         params.RequestId,
 			ServerId:          params.ServerId,
 			HasError:          params.HasError > 0,
--- a/web/public/js/components/server/http-access-log-partitions-box.js
+++ b/web/public/js/components/server/http-access-log-partitions-box.js
@@ -0,0 +1,60 @@
+Vue.component("http-access-log-partitions-box", {
+	props: ["v-partition", "v-day"],
+	mounted: function () {
+		let that = this
+		Tea.action("/servers/logs/partitionData")
+			.params({
+				day: this.vDay
+			})
+			.success(function (resp) {
+				that.partitions = []
+				resp.data.partitions.reverse().forEach(function (v) {
+					that.partitions.push({
+						code: v,
+						isDisabled: false
+					})
+				})
+				if (that.partitions.length > 0) {
+					if (that.vPartition == null || that.vPartition < 0) {
+						that.selectedPartition = that.partitions[0].code
+					}
+				}
+			})
+			.post()
+	},
+	data: function () {
+		return {
+			partitions: [],
+			selectedPartition: this.vPartition
+		}
+	},
+	methods: {
+		url: function (p) {
+			let u = window.location.toString()
+			u = u.replace(/\?partition=\d+/, "?")
+			u = u.replace(/\?requestId=\d+/, "?")
+			u = u.replace(/&partition=\d+/, "")
+			u = u.replace(/&requestId=\d+/, "")
+			if (u.indexOf("?") > 0) {
+				u += "&partition=" + p
+			} else {
+				u += "?partition=" + p
+			}
+			return u
+		},
+		disable: function (partition) {
+			this.partitions.forEach(function (p) {
+				if (p.code == partition) {
+					p.isDisabled = true
+				}
+			})
+		}
+	},
+	template: `<div v-if="partitions.length > 1">
+	<div class="ui divider" style="margin-bottom: 0"></div>
+	<div class="ui menu text small blue" style="margin-bottom: 0; margin-top: 0">
+		<a v-for="p in partitions" :href="url(p.code)" class="item" :class="{active: selectedPartition == p.code, disabled: p.isDisabled}">分表{{p.code}}</a>
+	</div>
+	<div class="ui divider" style="margin-top: 0"></div>
+</div>`
+})
--- a/web/views/@default/servers/logs/index.html
+++ b/web/views/@default/servers/logs/index.html
@@ -28,6 +28,8 @@
    </http-access-log-search-box>
 </form>

+<http-access-log-partitions-box :v-partition="partition" :v-day="day"></http-access-log-partitions-box>
+
 <warning-message v-if="isSlowQuery">看起来你的访问日志查询非常慢（{{slowQueryCost}}s），建议<span v-if="domain.length == 0"> 1）指定具体域名查询；2）</span>通过添加新的 <a href="/db">[日志节点]</a> 来分散存储访问日志。</warning-message>

 <p class="comment" v-if="accessLogs.length == 0">暂时还没有访问日志。</p>
@@ -40,10 +42,10 @@
 </table>

 <div v-if="accessLogs.length > 0">
-    <a :href="path + '?clusterId=' + clusterId + '&nodeId=' + nodeId + '&serverId=' + serverId + '&requestId=' + lastRequestId + '&day=' + day + '&hasError=' + hasError + '&hasWAF=' + hasWAF + '&keyword=' + keyword + '&ip=' + ip + '&domain=' + domain + '&hour=' + hour + '&pageSize=' + pageSize" v-if="hasPrev">上一页</a>
+    <a :href="path + '?clusterId=' + clusterId + '&nodeId=' + nodeId + '&serverId=' + serverId + '&requestId=' + lastRequestId + '&day=' + day + '&hasError=' + hasError + '&hasWAF=' + hasWAF + '&keyword=' + keyword + '&ip=' + ip + '&domain=' + domain + '&hour=' + hour + '&pageSize=' + pageSize + '&partition=' + partition" v-if="hasPrev">上一页</a>
    <span v-else class="disabled">上一页</span>
    <span class="disabled">&nbsp; | &nbsp;</span>
-    <a :href="path + '?clusterId=' + clusterId + '&nodeId=' + nodeId + '&serverId=' + serverId + '&requestId=' + nextRequestId + '&day=' + day + '&hasError=' + hasError + '&hasWAF=' + hasWAF + '&keyword=' + keyword + '&ip=' + ip + '&domain=' + domain + '&hour=' + hour + '&pageSize=' + pageSize" v-if="hasMore">下一页</a>
+    <a :href="path + '?clusterId=' + clusterId + '&nodeId=' + nodeId + '&serverId=' + serverId + '&requestId=' + nextRequestId + '&day=' + day + '&hasError=' + hasError + '&hasWAF=' + hasWAF + '&keyword=' + keyword + '&ip=' + ip + '&domain=' + domain + '&hour=' + hour + '&pageSize=' + pageSize + '&partition=' + partition" v-if="hasMore">下一页</a>
    <span v-else class="disabled">下一页</span>

    <page-size-selector></page-size-selector>