diff --git a/internal/accesslogs/storage_base.go b/internal/accesslogs/storage_base.go index deb6abb9..f125d617 100644 --- a/internal/accesslogs/storage_base.go +++ b/internal/accesslogs/storage_base.go @@ -12,8 +12,9 @@ import ( ) type BaseStorage struct { - isOk bool - version int + isOk bool + version int + firewallOnly bool } func (this *BaseStorage) SetVersion(version int) { @@ -32,6 +33,10 @@ func (this *BaseStorage) SetOk(isOk bool) { this.isOk = isOk } +func (this *BaseStorage) SetFirewallOnly(firewallOnly bool) { + this.firewallOnly = firewallOnly +} + // Marshal 对日志进行编码 func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) { return json.Marshal(accessLog) diff --git a/internal/accesslogs/storage_command.go b/internal/accesslogs/storage_command.go index 44f6f1f1..c11b17ab 100644 --- a/internal/accesslogs/storage_command.go +++ b/internal/accesslogs/storage_command.go @@ -61,6 +61,10 @@ func (this *CommandStorage) Write(accessLogs []*pb.HTTPAccessLog) error { return err } for _, accessLog := range accessLogs { + if this.firewallOnly && accessLog.FirewallPolicyId == 0 { + continue + } + data, err := this.Marshal(accessLog) if err != nil { logs.Error(err) diff --git a/internal/accesslogs/storage_es.go b/internal/accesslogs/storage_es.go index 7cff2e33..1ef26179 100644 --- a/internal/accesslogs/storage_es.go +++ b/internal/accesslogs/storage_es.go @@ -59,6 +59,10 @@ func (this *ESStorage) Write(accessLogs []*pb.HTTPAccessLog) error { indexName := this.FormatVariables(this.config.Index) typeName := this.FormatVariables(this.config.MappingType) for _, accessLog := range accessLogs { + if this.firewallOnly && accessLog.FirewallPolicyId == 0 { + continue + } + if len(accessLog.RequestId) == 0 { continue } diff --git a/internal/accesslogs/storage_file.go b/internal/accesslogs/storage_file.go index e3f3d90b..c5345ca6 100644 --- a/internal/accesslogs/storage_file.go +++ b/internal/accesslogs/storage_file.go @@ -57,6 +57,9 @@ func (this *FileStorage) Write(accessLogs []*pb.HTTPAccessLog) error { defer this.writeLocker.Unlock() for _, accessLog := range accessLogs { + if this.firewallOnly && accessLog.FirewallPolicyId == 0 { + continue + } data, err := this.Marshal(accessLog) if err != nil { logs.Error(err) diff --git a/internal/accesslogs/storage_interface.go b/internal/accesslogs/storage_interface.go index 810058db..9d052b6e 100644 --- a/internal/accesslogs/storage_interface.go +++ b/internal/accesslogs/storage_interface.go @@ -12,6 +12,9 @@ type StorageInterface interface { // SetVersion 设置版本 SetVersion(version int) + // SetFirewallOnly 设置是否只处理防火墙相关的访问日志 + SetFirewallOnly(firewallOnly bool) + IsOk() bool SetOk(ok bool) diff --git a/internal/accesslogs/storage_manager.go b/internal/accesslogs/storage_manager.go index 15108da8..4c6b2b4f 100644 --- a/internal/accesslogs/storage_manager.go +++ b/internal/accesslogs/storage_manager.go @@ -101,6 +101,7 @@ func (this *StorageManager) Loop() error { } storage.SetVersion(types.Int(policy.Version)) + storage.SetFirewallOnly(policy.FirewallOnly == 1) err := storage.Start() if err != nil { remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error()) @@ -116,6 +117,7 @@ func (this *StorageManager) Loop() error { continue } storage.SetVersion(types.Int(policy.Version)) + storage.SetFirewallOnly(policy.FirewallOnly == 1) this.storageMap[policyId] = storage err = storage.Start() if err != nil { diff --git a/internal/accesslogs/storage_syslog.go b/internal/accesslogs/storage_syslog.go index 18b9591f..4ddc60e7 100644 --- a/internal/accesslogs/storage_syslog.go +++ b/internal/accesslogs/storage_syslog.go @@ -106,6 +106,9 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error { } for _, accessLog := range accessLogs { + if this.firewallOnly && accessLog.FirewallPolicyId == 0 { + continue + } data, err := this.Marshal(accessLog) if err != nil { logs.Error(err) diff --git a/internal/accesslogs/storage_tcp.go b/internal/accesslogs/storage_tcp.go index 77a85eed..c21ab235 100644 --- a/internal/accesslogs/storage_tcp.go +++ b/internal/accesslogs/storage_tcp.go @@ -60,6 +60,9 @@ func (this *TCPStorage) Write(accessLogs []*pb.HTTPAccessLog) error { defer this.writeLocker.Unlock() for _, accessLog := range accessLogs { + if this.firewallOnly && accessLog.FirewallPolicyId == 0 { + continue + } data, err := this.Marshal(accessLog) if err != nil { logs.Error(err) diff --git a/internal/db/models/http_access_log_policy_dao.go b/internal/db/models/http_access_log_policy_dao.go index d0b9379b..8843a990 100644 --- a/internal/db/models/http_access_log_policy_dao.go +++ b/internal/db/models/http_access_log_policy_dao.go @@ -1,13 +1,10 @@ package models import ( - "bytes" - "encoding/json" "github.com/TeaOSLab/EdgeAPI/internal/errors" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" - "github.com/iwind/TeaGo/maps" ) const ( @@ -109,7 +106,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res } // CreatePolicy 创建策略 -func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool) (policyId int64, err error) { +func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool) (policyId int64, err error) { var op = NewHTTPAccessLogPolicyOperator() op.Name = name op.Type = policyType @@ -121,12 +118,13 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy } op.IsPublic = isPublic op.IsOn = true + op.FirewallOnly = firewallOnly op.State = HTTPAccessLogPolicyStateEnabled return this.SaveInt64(tx, op) } // UpdatePolicy 修改策略 -func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, isOn bool) error { +func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, isOn bool) error { if policyId <= 0 { return errors.New("invalid policyId") } @@ -140,7 +138,6 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam if oldOne == nil { return nil } - var oldPolicy = oldOne.(*HTTPAccessLogPolicy) var op = NewHTTPAccessLogPolicyOperator() op.Id = policyId @@ -156,22 +153,11 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam op.Conds = "{}" } - // 版本号 - if len(oldPolicy.Options) == 0 || len(optionsJSON) == 0 { - op.Version = dbs.SQL("version+1") - } else { - var m1 = maps.Map{} - _ = json.Unmarshal(oldPolicy.Options, &m1) - - var m2 = maps.Map{} - _ = json.Unmarshal(optionsJSON, &m2) - - if bytes.Compare(m1.AsJSON(), m2.AsJSON()) != 0 { - op.Version = dbs.SQL("version+1") - } - } + // 版本号总是加1 + op.Version = dbs.SQL("version+1") op.IsPublic = isPublic + op.FirewallOnly = firewallOnly op.IsOn = isOn return this.Save(tx, op) } diff --git a/internal/db/models/http_access_log_policy_model.go b/internal/db/models/http_access_log_policy_model.go index 8d444d63..fea9184c 100644 --- a/internal/db/models/http_access_log_policy_model.go +++ b/internal/db/models/http_access_log_policy_model.go @@ -4,35 +4,37 @@ import "github.com/iwind/TeaGo/dbs" // HTTPAccessLogPolicy 访问日志策略 type HTTPAccessLogPolicy struct { - Id uint32 `field:"id"` // ID - TemplateId uint32 `field:"templateId"` // 模版ID - AdminId uint32 `field:"adminId"` // 管理员ID - UserId uint32 `field:"userId"` // 用户ID - State uint8 `field:"state"` // 状态 - CreatedAt uint64 `field:"createdAt"` // 创建时间 - Name string `field:"name"` // 名称 - IsOn bool `field:"isOn"` // 是否启用 - Type string `field:"type"` // 存储类型 - Options dbs.JSON `field:"options"` // 存储选项 - Conds dbs.JSON `field:"conds"` // 请求条件 - IsPublic bool `field:"isPublic"` // 是否为公用 - Version uint32 `field:"version"` // 版本号 + Id uint32 `field:"id"` // ID + TemplateId uint32 `field:"templateId"` // 模版ID + AdminId uint32 `field:"adminId"` // 管理员ID + UserId uint32 `field:"userId"` // 用户ID + State uint8 `field:"state"` // 状态 + CreatedAt uint64 `field:"createdAt"` // 创建时间 + Name string `field:"name"` // 名称 + IsOn bool `field:"isOn"` // 是否启用 + Type string `field:"type"` // 存储类型 + Options dbs.JSON `field:"options"` // 存储选项 + Conds dbs.JSON `field:"conds"` // 请求条件 + IsPublic bool `field:"isPublic"` // 是否为公用 + FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关 + Version uint32 `field:"version"` // 版本号 } type HTTPAccessLogPolicyOperator struct { - Id interface{} // ID - TemplateId interface{} // 模版ID - AdminId interface{} // 管理员ID - UserId interface{} // 用户ID - State interface{} // 状态 - CreatedAt interface{} // 创建时间 - Name interface{} // 名称 - IsOn interface{} // 是否启用 - Type interface{} // 存储类型 - Options interface{} // 存储选项 - Conds interface{} // 请求条件 - IsPublic interface{} // 是否为公用 - Version interface{} // 版本号 + Id interface{} // ID + TemplateId interface{} // 模版ID + AdminId interface{} // 管理员ID + UserId interface{} // 用户ID + State interface{} // 状态 + CreatedAt interface{} // 创建时间 + Name interface{} // 名称 + IsOn interface{} // 是否启用 + Type interface{} // 存储类型 + Options interface{} // 存储选项 + Conds interface{} // 请求条件 + IsPublic interface{} // 是否为公用 + FirewallOnly interface{} // 是否只记录防火墙相关 + Version interface{} // 版本号 } func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator { diff --git a/internal/rpc/services/service_http_access_log_policy.go b/internal/rpc/services/service_http_access_log_policy.go index f303e564..c43feb02 100644 --- a/internal/rpc/services/service_http_access_log_policy.go +++ b/internal/rpc/services/service_http_access_log_policy.go @@ -41,13 +41,14 @@ func (this *HTTPAccessLogPolicyService) ListEnabledHTTPAccessLogPolicies(ctx con var pbPolicies = []*pb.HTTPAccessLogPolicy{} for _, policy := range policies { pbPolicies = append(pbPolicies, &pb.HTTPAccessLogPolicy{ - Id: int64(policy.Id), - Name: policy.Name, - IsOn: policy.IsOn, - Type: policy.Type, - OptionsJSON: policy.Options, - CondsJSON: policy.Conds, - IsPublic: policy.IsPublic, + Id: int64(policy.Id), + Name: policy.Name, + IsOn: policy.IsOn, + Type: policy.Type, + OptionsJSON: policy.Options, + CondsJSON: policy.Conds, + IsPublic: policy.IsPublic, + FirewallOnly: policy.FirewallOnly == 1, }) } return &pb.ListEnabledHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil @@ -71,7 +72,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co } // 创建 - policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic) + policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly) if err != nil { return nil, err } @@ -96,7 +97,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co } // 保存修改 - err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.IsOn) + err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.IsOn) if err != nil { return nil, err } @@ -119,13 +120,14 @@ func (this *HTTPAccessLogPolicyService) FindEnabledHTTPAccessLogPolicy(ctx conte return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: nil}, nil } return &pb.FindEnabledHTTPAccessLogPolicyResponse{HttpAccessLogPolicy: &pb.HTTPAccessLogPolicy{ - Id: int64(policy.Id), - Name: policy.Name, - IsOn: policy.IsOn, - Type: policy.Type, - OptionsJSON: policy.Options, - CondsJSON: policy.Conds, - IsPublic: policy.IsPublic, + Id: int64(policy.Id), + Name: policy.Name, + IsOn: policy.IsOn, + Type: policy.Type, + OptionsJSON: policy.Options, + CondsJSON: policy.Conds, + IsPublic: policy.IsPublic, + FirewallOnly: policy.FirewallOnly == 1, }}, nil }