TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-04 16:00:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

访问日志策略增加只记录WAF相关访问日志选项

Browse Source
This commit is contained in:
GoEdgeLab
2022-04-22 17:13:59 +08:00
parent fe520744b0
commit 0dd3d6d5d3
11 changed files with 81 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/accesslogs/storage_base.go
View File

@@ -12,8 +12,9 @@ import (
) )
type BaseStorage struct { type BaseStorage struct {
isOk bool isOk bool
version int version int
firewallOnly bool
} }
func (this *BaseStorage) SetVersion(version int) { func (this *BaseStorage) SetVersion(version int) {
@@ -32,6 +33,10 @@ func (this *BaseStorage) SetOk(isOk bool) {
this.isOk = isOk this.isOk = isOk
} }
func (this *BaseStorage) SetFirewallOnly(firewallOnly bool) {
this.firewallOnly = firewallOnly
}
// Marshal 对日志进行编码 // Marshal 对日志进行编码
func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) { func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
return json.Marshal(accessLog) return json.Marshal(accessLog)

4
internal/accesslogs/storage_command.go
View File

@@ -61,6 +61,10 @@ func (this *CommandStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
return err return err
} }
for _, accessLog := range accessLogs { for _, accessLog := range accessLogs {
if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
continue
}
data, err := this.Marshal(accessLog) data, err := this.Marshal(accessLog)
if err != nil { if err != nil {
logs.Error(err) logs.Error(err)

4
internal/accesslogs/storage_es.go
View File

@@ -59,6 +59,10 @@ func (this *ESStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
indexName := this.FormatVariables(this.config.Index) indexName := this.FormatVariables(this.config.Index)
typeName := this.FormatVariables(this.config.MappingType) typeName := this.FormatVariables(this.config.MappingType)
for _, accessLog := range accessLogs { for _, accessLog := range accessLogs {
if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
continue
}
if len(accessLog.RequestId) == 0 { if len(accessLog.RequestId) == 0 {
continue continue
} }

3
internal/accesslogs/storage_file.go
View File

@@ -57,6 +57,9 @@ func (this *FileStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
defer this.writeLocker.Unlock() defer this.writeLocker.Unlock()
for _, accessLog := range accessLogs { for _, accessLog := range accessLogs {
if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
continue
}
data, err := this.Marshal(accessLog) data, err := this.Marshal(accessLog)
if err != nil { if err != nil {
logs.Error(err) logs.Error(err)

3
internal/accesslogs/storage_interface.go
View File

@@ -12,6 +12,9 @@ type StorageInterface interface {
// SetVersion 设置版本 // SetVersion 设置版本
SetVersion(version int) SetVersion(version int)
// SetFirewallOnly 设置是否只处理防火墙相关的访问日志
SetFirewallOnly(firewallOnly bool)
IsOk() bool IsOk() bool
SetOk(ok bool) SetOk(ok bool)

2
internal/accesslogs/storage_manager.go
View File

@@ -101,6 +101,7 @@ func (this *StorageManager) Loop() error {
} }
storage.SetVersion(types.Int(policy.Version)) storage.SetVersion(types.Int(policy.Version))
storage.SetFirewallOnly(policy.FirewallOnly == 1)
err := storage.Start() err := storage.Start()
if err != nil { if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error()) remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error())
@@ -116,6 +117,7 @@ func (this *StorageManager) Loop() error {
continue continue
} }
storage.SetVersion(types.Int(policy.Version)) storage.SetVersion(types.Int(policy.Version))
storage.SetFirewallOnly(policy.FirewallOnly == 1)
this.storageMap[policyId] = storage this.storageMap[policyId] = storage
err = storage.Start() err = storage.Start()
if err != nil { if err != nil {

3
internal/accesslogs/storage_syslog.go
View File

@@ -106,6 +106,9 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
} }
for _, accessLog := range accessLogs { for _, accessLog := range accessLogs {
if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
continue
}
data, err := this.Marshal(accessLog) data, err := this.Marshal(accessLog)
if err != nil { if err != nil {
logs.Error(err) logs.Error(err)

3
internal/accesslogs/storage_tcp.go
View File

@@ -60,6 +60,9 @@ func (this *TCPStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
defer this.writeLocker.Unlock() defer this.writeLocker.Unlock()
for _, accessLog := range accessLogs { for _, accessLog := range accessLogs {
if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
continue
}
data, err := this.Marshal(accessLog) data, err := this.Marshal(accessLog)
if err != nil { if err != nil {
logs.Error(err) logs.Error(err)

26
internal/db/models/http_access_log_policy_dao.go
View File

@@ -1,13 +1,10 @@
package models package models
import ( import (
"bytes"
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/errors" "github.com/TeaOSLab/EdgeAPI/internal/errors"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
) )
const ( const (
@@ -109,7 +106,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
} }
// CreatePolicy 创建策略 // CreatePolicy 创建策略
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool) (policyId int64, err error) { func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool) (policyId int64, err error) {
var op = NewHTTPAccessLogPolicyOperator() var op = NewHTTPAccessLogPolicyOperator()
op.Name = name op.Name = name
op.Type = policyType op.Type = policyType
@@ -121,12 +118,13 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
} }
op.IsPublic = isPublic op.IsPublic = isPublic
op.IsOn = true op.IsOn = true
op.FirewallOnly = firewallOnly
op.State = HTTPAccessLogPolicyStateEnabled op.State = HTTPAccessLogPolicyStateEnabled
return this.SaveInt64(tx, op) return this.SaveInt64(tx, op)
} }
// UpdatePolicy 修改策略 // UpdatePolicy 修改策略
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, isOn bool) error { func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, isOn bool) error {
if policyId <= 0 { if policyId <= 0 {
return errors.New("invalid policyId") return errors.New("invalid policyId")
} }
@@ -140,7 +138,6 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
if oldOne == nil { if oldOne == nil {
return nil return nil
} }
var oldPolicy = oldOne.(*HTTPAccessLogPolicy)
var op = NewHTTPAccessLogPolicyOperator() var op = NewHTTPAccessLogPolicyOperator()
op.Id = policyId op.Id = policyId
@@ -156,22 +153,11 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
op.Conds = "{}" op.Conds = "{}"
} }
// 版本号 // 版本号总是加1
if len(oldPolicy.Options) == 0 || len(optionsJSON) == 0 { op.Version = dbs.SQL("version+1")
op.Version = dbs.SQL("version+1")
} else {
var m1 = maps.Map{}
_ = json.Unmarshal(oldPolicy.Options, &m1)
var m2 = maps.Map{}
_ = json.Unmarshal(optionsJSON, &m2)
if bytes.Compare(m1.AsJSON(), m2.AsJSON()) != 0 {
op.Version = dbs.SQL("version+1")
}
}
op.IsPublic = isPublic op.IsPublic = isPublic
op.FirewallOnly = firewallOnly
op.IsOn = isOn op.IsOn = isOn
return this.Save(tx, op) return this.Save(tx, op)
} }

54
internal/db/models/http_access_log_policy_model.go
View File

@@ -4,35 +4,37 @@ import "github.com/iwind/TeaGo/dbs"
// HTTPAccessLogPolicy 访问日志策略 // HTTPAccessLogPolicy 访问日志策略
type HTTPAccessLogPolicy struct { type HTTPAccessLogPolicy struct {
Id uint32 `field:"id"` // ID Id uint32 `field:"id"` // ID
TemplateId uint32 `field:"templateId"` // 模版ID TemplateId uint32 `field:"templateId"` // 模版ID
AdminId uint32 `field:"adminId"` // 管理员ID AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID UserId uint32 `field:"userId"` // 用户ID
State uint8 `field:"state"` // 状态 State uint8 `field:"state"` // 状态
CreatedAt uint64 `field:"createdAt"` // 创建时间 CreatedAt uint64 `field:"createdAt"` // 创建时间
Name string `field:"name"` // 名称 Name string `field:"name"` // 名称
IsOn bool `field:"isOn"` // 是否启用 IsOn bool `field:"isOn"` // 是否启用
Type string `field:"type"` // 存储类型 Type string `field:"type"` // 存储类型
Options dbs.JSON `field:"options"` // 存储选项 Options dbs.JSON `field:"options"` // 存储选项
Conds dbs.JSON `field:"conds"` // 请求条件 Conds dbs.JSON `field:"conds"` // 请求条件
IsPublic bool `field:"isPublic"` // 是否为公用 IsPublic bool `field:"isPublic"` // 是否为公用
Version uint32 `field:"version"` // 版本号 FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关
Version uint32 `field:"version"` // 版本号
} }
type HTTPAccessLogPolicyOperator struct { type HTTPAccessLogPolicyOperator struct {
Id interface{} // ID Id interface{} // ID
TemplateId interface{} // 模版ID TemplateId interface{} // 模版ID
AdminId interface{} // 管理员ID AdminId interface{} // 管理员ID
UserId interface{} // 用户ID UserId interface{} // 用户ID
State interface{} // 状态 State interface{} // 状态
CreatedAt interface{} // 创建时间 CreatedAt interface{} // 创建时间
Name interface{} // 名称 Name interface{} // 名称
IsOn interface{} // 是否启用 IsOn interface{} // 是否启用
Type interface{} // 存储类型 Type interface{} // 存储类型
Options interface{} // 存储选项 Options interface{} // 存储选项
Conds interface{} // 请求条件 Conds interface{} // 请求条件
IsPublic interface{} // 是否为公用 IsPublic interface{} // 是否为公用
Version interface{} // 版本号 FirewallOnly interface{} // 是否只记录防火墙相关
Version interface{} // 版本号
} }
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator { func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

34
internal/rpc/services/service_http_access_log_policy.go
View File

@@ -41,13 +41,14 @@ func (this *HTTPAccessLogPolicyService) ListEnabledHTTPAccessLogPolicies(ctx con
var pbPolicies = []*pb.HTTPAccessLogPolicy{} var pbPolicies = []*pb.HTTPAccessLogPolicy{}
for _, policy := range policies { for _, policy := range policies {
pbPolicies = append(pbPolicies, &pb.HTTPAccessLogPolicy{ pbPolicies = append(pbPolicies, &pb.HTTPAccessLogPolicy{
Id: int64(policy.Id), Id: int64(policy.Id),
Name: policy.Name, Name: policy.Name,
IsOn: policy.IsOn, IsOn: policy.IsOn,
Type: policy.Type, Type: policy.Type,
OptionsJSON: policy.Options, OptionsJSON: policy.Options,
CondsJSON: policy.Conds, CondsJSON: policy.Conds,
IsPublic: policy.IsPublic, IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
}) })
} }
return &pb.ListEnabledHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil return &pb.ListEnabledHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -71,7 +72,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
} }
// 创建 // 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic) policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -96,7 +97,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
} }
// 保存修改 // 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.IsOn) err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.IsOn)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -119,13 +120,14 @@ func (this *HTTPAccessLogPolicyService) FindEnabledHTTPAccessLogPolicy(ctx conte
return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: nil}, nil return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: nil}, nil
} }
return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: &pb.HTTPAccessLogPolicy{ return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: &pb.HTTPAccessLogPolicy{
Id: int64(policy.Id), Id: int64(policy.Id),
Name: policy.Name, Name: policy.Name,
IsOn: policy.IsOn, IsOn: policy.IsOn,
Type: policy.Type, Type: policy.Type,
OptionsJSON: policy.Options, OptionsJSON: policy.Options,
CondsJSON: policy.Conds, CondsJSON: policy.Conds,
IsPublic: policy.IsPublic, IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
}}, nil }}, nil
} }