实现IP库制品管理API、自动更新程序

This commit is contained in:
GoEdgeLab
2022-08-23 14:54:53 +08:00
parent 5b14b02df8
commit f230cf410f
11 changed files with 5157 additions and 3303 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -12,18 +12,23 @@ import (
//go:embed internal-ip-library.db //go:embed internal-ip-library.db
var ipLibraryData []byte var ipLibraryData []byte
var library = NewIPLibrary() var defaultLibrary = NewIPLibrary()
func Init() error { func DefaultIPLibraryData() []byte {
return library.Init() return ipLibraryData
}
func InitDefault() error {
defaultLibrary.reader = nil
return defaultLibrary.InitFromData(ipLibraryData)
} }
func Lookup(ip net.IP) *QueryResult { func Lookup(ip net.IP) *QueryResult {
return library.Lookup(ip) return defaultLibrary.Lookup(ip)
} }
func LookupIP(ip string) *QueryResult { func LookupIP(ip string) *QueryResult {
return library.LookupIP(ip) return defaultLibrary.LookupIP(ip)
} }
type IPLibrary struct { type IPLibrary struct {
@@ -34,11 +39,15 @@ func NewIPLibrary() *IPLibrary {
return &IPLibrary{} return &IPLibrary{}
} }
func (this *IPLibrary) Init() error { func NewIPLibraryWithReader(reader *Reader) *IPLibrary {
if len(ipLibraryData) == 0 || this.reader != nil { return &IPLibrary{reader: reader}
}
func (this *IPLibrary) InitFromData(data []byte) error {
if len(data) == 0 || this.reader != nil {
return nil return nil
} }
var reader = bytes.NewReader(ipLibraryData) var reader = bytes.NewReader(data)
gzipReader, err := gzip.NewReader(reader) gzipReader, err := gzip.NewReader(reader)
if err != nil { if err != nil {
return err return err

View File

@@ -15,7 +15,7 @@ import (
func TestIPLibrary_Init(t *testing.T) { func TestIPLibrary_Init(t *testing.T) {
var lib = iplibrary.NewIPLibrary() var lib = iplibrary.NewIPLibrary()
err := lib.Init() err := lib.InitFromData(iplibrary.DefaultIPLibraryData())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -29,7 +29,7 @@ func TestIPLibrary_Lookup(t *testing.T) {
var before = time.Now() var before = time.Now()
err := lib.Init() err := lib.InitFromData(iplibrary.DefaultIPLibraryData())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -59,7 +59,7 @@ func TestIPLibrary_Lookup(t *testing.T) {
func TestIPLibrary_LookupIP(t *testing.T) { func TestIPLibrary_LookupIP(t *testing.T) {
var lib = iplibrary.NewIPLibrary() var lib = iplibrary.NewIPLibrary()
err := lib.Init() err := lib.InitFromData(iplibrary.DefaultIPLibraryData())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -78,7 +78,7 @@ func TestIPLibrary_LookupIP(t *testing.T) {
func BenchmarkIPLibrary_Lookup(b *testing.B) { func BenchmarkIPLibrary_Lookup(b *testing.B) {
var lib = iplibrary.NewIPLibrary() var lib = iplibrary.NewIPLibrary()
err := lib.Init() err := lib.InitFromData(iplibrary.DefaultIPLibraryData())
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }

View File

@@ -34,6 +34,7 @@ type Provider struct {
type Meta struct { type Meta struct {
Version int `json:"version"` // IP库版本 Version int `json:"version"` // IP库版本
Code string `json:"code"` // 代号用来区分不同的IP库
Author string `json:"author"` Author string `json:"author"`
Countries []*Country `json:"countries"` Countries []*Country `json:"countries"`
Provinces []*Province `json:"provinces"` Provinces []*Province `json:"provinces"`

View File

@@ -4,6 +4,8 @@ package iplibrary
import ( import (
"compress/gzip" "compress/gzip"
"errors"
"io"
"net" "net"
"os" "os"
) )
@@ -21,9 +23,13 @@ func NewFileReader(path string) (*FileReader, error) {
_ = fp.Close() _ = fp.Close()
}() }()
gzReader, err := gzip.NewReader(fp) return NewFileDataReader(fp)
}
func NewFileDataReader(dataReader io.Reader) (*FileReader, error) {
gzReader, err := gzip.NewReader(dataReader)
if err != nil { if err != nil {
return nil, err return nil, errors.New("create gzip reader failed: " + err.Error())
} }
reader, err := NewReader(gzReader) reader, err := NewReader(gzReader)
@@ -43,3 +49,7 @@ func (this *FileReader) Meta() *Meta {
func (this *FileReader) Lookup(ip net.IP) *QueryResult { func (this *FileReader) Lookup(ip net.IP) *QueryResult {
return this.rawReader.Lookup(ip) return this.rawReader.Lookup(ip)
} }
func (this *FileReader) RawReader() *Reader {
return this.rawReader
}

268
pkg/iplibrary/updater.go Normal file
View File

@@ -0,0 +1,268 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package iplibrary
import (
"errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"io"
"os"
"strings"
"time"
)
type UpdaterSource interface {
// DataDir 文件目录
DataDir() string
// FindLatestFile 检查最新的IP库文件
FindLatestFile() (code string, fileId int64, err error)
// DownloadFile 下载文件
DownloadFile(fileId int64, writer io.Writer) error
// LogInfo 普通日志
LogInfo(message string)
// LogError 错误日志
LogError(err error)
}
type Updater struct {
source UpdaterSource
currentCode string
ticker *time.Ticker
isUpdating bool
}
func NewUpdater(source UpdaterSource, interval time.Duration) *Updater {
return &Updater{
source: source,
ticker: time.NewTicker(interval),
}
}
func (this *Updater) Start() {
// 初始化
err := this.Init()
if err != nil {
this.source.LogError(err)
}
// 先运行一次
err = this.Loop()
if err != nil {
this.source.LogError(err)
}
// 开始定时运行
for range this.ticker.C {
err = this.Loop()
if err != nil {
this.source.LogError(err)
}
}
}
func (this *Updater) Init() error {
// 检查当前正在使用的IP库
var path = this.source.DataDir() + "/ip-library.db"
fp, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return errors.New("read ip library file failed '" + err.Error() + "'")
}
defer func() {
_ = fp.Close()
}()
return this.loadFile(fp)
}
func (this *Updater) Loop() error {
if this.isUpdating {
return nil
}
this.isUpdating = true
defer func() {
this.isUpdating = false
}()
code, fileId, err := this.source.FindLatestFile()
if err != nil {
// 不提示连接错误
if this.isConnError(err) {
return nil
}
return err
}
if len(code) == 0 || fileId <= 0 {
// 还原到内置IP库
if len(this.currentCode) > 0 {
this.currentCode = ""
this.source.LogInfo("resetting to default ip library ...")
var defaultPath = this.source.DataDir() + "/ip-library.db"
_, err = os.Stat(defaultPath)
if err == nil {
err = os.Remove(defaultPath)
if err != nil {
this.source.LogError(errors.New("can not remove default 'ip-library.db'"))
}
}
err = InitDefault()
if err != nil {
this.source.LogError(errors.New("initialize default ip library failed: " + err.Error()))
}
}
return nil
}
// 下载
if this.currentCode == code {
// 不再重复下载
return nil
}
// 检查是否存在
var dir = this.source.DataDir()
var path = dir + "/ip-" + code + ".db"
stat, err := os.Stat(path)
if err == nil && !stat.IsDir() && stat.Size() > 0 {
fp, err := os.Open(path)
if err != nil {
return err
}
defer func() {
_ = fp.Close()
}()
err = this.loadFile(fp)
if err != nil {
// 尝试删除
_ = os.Remove(path)
} else {
this.currentCode = code
// 拷贝到 ip-library.db
err = this.createDefaultFile(path, dir)
if err != nil {
this.source.LogError(err)
}
}
return err
}
// write to file
fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
return errors.New("create ip library file failed: " + err.Error())
}
var isOk = false
defer func() {
if !isOk {
_ = os.Remove(path)
}
}()
err = this.source.DownloadFile(fileId, fp)
if err != nil {
_ = fp.Close()
return err
}
err = fp.Close()
if err != nil {
return nil
}
// load library from file
fp, err = os.Open(path)
if err != nil {
return nil
}
err = this.loadFile(fp)
_ = fp.Close()
if err != nil {
return errors.New("load file failed: " + err.Error())
}
isOk = true
this.currentCode = code
// 拷贝到 ip-library.db
err = this.createDefaultFile(path, dir)
if err != nil {
this.source.LogError(err)
}
return nil
}
func (this *Updater) loadFile(fp *os.File) error {
this.source.LogInfo("load ip library from '" + fp.Name() + "' ...")
fileReader, err := NewFileDataReader(fp)
if err != nil {
return errors.New("load ip library from reader failed: " + err.Error())
}
var reader = fileReader.RawReader()
defaultLibrary = NewIPLibraryWithReader(reader)
this.currentCode = reader.Meta().Code
return nil
}
func (this *Updater) createDefaultFile(sourcePath string, dir string) error {
sourceFp, err := os.Open(sourcePath)
if err != nil {
return errors.New("prepare to copy file to 'ip-library.db' failed: " + err.Error())
}
defer func() {
_ = sourceFp.Close()
}()
dstFp, err := os.Create(dir + "/ip-library.db")
if err != nil {
return errors.New("prepare to copy file to 'ip-library.db' failed: " + err.Error())
}
defer func() {
_ = dstFp.Close()
}()
_, err = io.Copy(dstFp, sourceFp)
if err != nil {
return errors.New("copy file to 'ip-library.db' failed: " + err.Error())
}
return nil
}
// isConnError 是否为连接错误
func (this *Updater) isConnError(err error) bool {
if err == nil {
return false
}
// 检查是否为连接错误
statusErr, ok := status.FromError(err)
if ok {
var errorCode = statusErr.Code()
return errorCode == codes.Unavailable || errorCode == codes.Canceled
}
if strings.Contains(err.Error(), "code = Canceled") {
return true
}
return false
}

View File

@@ -0,0 +1,53 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package iplibrary_test
import (
"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io"
"testing"
"time"
)
type updaterSource struct {
t *testing.T
}
func (this *updaterSource) DataDir() string {
return Tea.Root + "/data"
}
func (this *updaterSource) FindLatestFile() (code string, fileId int64, err error) {
return "CODE", 1, nil
}
func (this *updaterSource) DownloadFile(fileId int64, writer io.Writer) error {
this.t.Log("downloading file:", fileId, "writer:", writer)
_, err := writer.Write(iplibrary.DefaultIPLibraryData())
return err
}
func (this *updaterSource) LogInfo(message string) {
this.t.Log(message)
}
func (this *updaterSource) LogError(err error) {
this.t.Fatal(err)
}
func TestNewUpdater(t *testing.T) {
var updater = iplibrary.NewUpdater(&updaterSource{
t: t,
}, 1*time.Minute)
err := updater.Init()
if err != nil {
t.Fatal(err)
}
err = updater.Loop()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,205 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.25.0
// protoc v3.19.4
// source: models/model_ip_library_artifact.proto
package pb
import (
proto "github.com/golang/protobuf/proto"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// This is a compile-time assertion that a sufficiently up-to-date version
// of the legacy proto package is being used.
const _ = proto.ProtoPackageIsVersion4
type IPLibraryArtifact struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
FileId int64 `protobuf:"varint,2,opt,name=fileId,proto3" json:"fileId,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
MetaJSON []byte `protobuf:"bytes,4,opt,name=metaJSON,proto3" json:"metaJSON,omitempty"`
IsPublic bool `protobuf:"varint,5,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"`
Code string `protobuf:"bytes,7,opt,name=code,proto3" json:"code,omitempty"`
}
func (x *IPLibraryArtifact) Reset() {
*x = IPLibraryArtifact{}
if protoimpl.UnsafeEnabled {
mi := &file_models_model_ip_library_artifact_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *IPLibraryArtifact) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*IPLibraryArtifact) ProtoMessage() {}
func (x *IPLibraryArtifact) ProtoReflect() protoreflect.Message {
mi := &file_models_model_ip_library_artifact_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use IPLibraryArtifact.ProtoReflect.Descriptor instead.
func (*IPLibraryArtifact) Descriptor() ([]byte, []int) {
return file_models_model_ip_library_artifact_proto_rawDescGZIP(), []int{0}
}
func (x *IPLibraryArtifact) GetId() int64 {
if x != nil {
return x.Id
}
return 0
}
func (x *IPLibraryArtifact) GetFileId() int64 {
if x != nil {
return x.FileId
}
return 0
}
func (x *IPLibraryArtifact) GetCreatedAt() int64 {
if x != nil {
return x.CreatedAt
}
return 0
}
func (x *IPLibraryArtifact) GetMetaJSON() []byte {
if x != nil {
return x.MetaJSON
}
return nil
}
func (x *IPLibraryArtifact) GetIsPublic() bool {
if x != nil {
return x.IsPublic
}
return false
}
func (x *IPLibraryArtifact) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *IPLibraryArtifact) GetCode() string {
if x != nil {
return x.Code
}
return ""
}
var File_models_model_ip_library_artifact_proto protoreflect.FileDescriptor
var file_models_model_ip_library_artifact_proto_rawDesc = []byte{
0x0a, 0x26, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x69,
0x70, 0x5f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61,
0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x70, 0x62, 0x22, 0xb9, 0x01, 0x0a,
0x11, 0x49, 0x50, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61,
0x63, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02,
0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01,
0x28, 0x03, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x72,
0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63,
0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61,
0x4a, 0x53, 0x4f, 0x4e, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63,
0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63,
0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01,
0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x2f, 0x70, 0x62,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_models_model_ip_library_artifact_proto_rawDescOnce sync.Once
file_models_model_ip_library_artifact_proto_rawDescData = file_models_model_ip_library_artifact_proto_rawDesc
)
func file_models_model_ip_library_artifact_proto_rawDescGZIP() []byte {
file_models_model_ip_library_artifact_proto_rawDescOnce.Do(func() {
file_models_model_ip_library_artifact_proto_rawDescData = protoimpl.X.CompressGZIP(file_models_model_ip_library_artifact_proto_rawDescData)
})
return file_models_model_ip_library_artifact_proto_rawDescData
}
var file_models_model_ip_library_artifact_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_models_model_ip_library_artifact_proto_goTypes = []interface{}{
(*IPLibraryArtifact)(nil), // 0: pb.IPLibraryArtifact
}
var file_models_model_ip_library_artifact_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_models_model_ip_library_artifact_proto_init() }
func file_models_model_ip_library_artifact_proto_init() {
if File_models_model_ip_library_artifact_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_models_model_ip_library_artifact_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IPLibraryArtifact); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_models_model_ip_library_artifact_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_models_model_ip_library_artifact_proto_goTypes,
DependencyIndexes: file_models_model_ip_library_artifact_proto_depIdxs,
MessageInfos: file_models_model_ip_library_artifact_proto_msgTypes,
}.Build()
File_models_model_ip_library_artifact_proto = out.File
file_models_model_ip_library_artifact_proto_rawDesc = nil
file_models_model_ip_library_artifact_proto_goTypes = nil
file_models_model_ip_library_artifact_proto_depIdxs = nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,14 @@
syntax = "proto3";
option go_package = "./pb";
package pb;
message IPLibraryArtifact {
int64 id = 1;
int64 fileId = 2;
int64 createdAt = 3;
bytes metaJSON = 4;
bool isPublic = 5;
string name = 6;
string code = 7;
}

View File

@@ -0,0 +1,77 @@
syntax = "proto3";
option go_package = "./pb";
package pb;
import "models/model_ip_library_artifact.proto";
import "models/rpc_messages.proto";
// IP库制品
service IPLibraryArtifactService {
// 创建制品
rpc createIPLibraryArtifact(CreateIPLibraryArtifactRequest) returns (CreateIPLibraryArtifactResponse);
// 使用/取消使用制品
rpc updateIPLibraryArtifactIsPublic(UpdateIPLibraryArtifactIsPublicRequest) returns (RPCSuccess);
// 查询所有制品
rpc findAllIPLibraryArtifacts(FindAllIPLibraryArtifactsRequest) returns (FindAllIPLibraryArtifactsResponse);
// 查找单个制品信息
rpc findIPLibraryArtifact(FindIPLibraryArtifactRequest) returns (FindIPLibraryArtifactResponse);
// 查找当前正在使用的制品
rpc findPublicIPLibraryArtifact(FindPublicIPLibraryArtifactRequest) returns (FindPublicIPLibraryArtifactResponse);
// 删除制品
rpc deleteIPLibraryArtifact(DeleteIPLibraryArtifactRequest) returns (RPCSuccess);
}
// 创建制品
message CreateIPLibraryArtifactRequest {
int64 fileId = 1;
bytes metaJSON = 2;
string name = 3;
}
message CreateIPLibraryArtifactResponse {
int64 ipLibraryArtifactId = 1;
}
// 使用/取消使用制品
message UpdateIPLibraryArtifactIsPublicRequest {
int64 ipLibraryArtifactId = 1;
bool isPublic = 2;
}
// 查询所有制品
message FindAllIPLibraryArtifactsRequest {
}
message FindAllIPLibraryArtifactsResponse {
repeated IPLibraryArtifact ipLibraryArtifacts = 1;
}
// 查找单个制品信息
message FindIPLibraryArtifactRequest {
int64 ipLibraryArtifactId = 1;
}
message FindIPLibraryArtifactResponse {
IPLibraryArtifact ipLibraryArtifact = 1;
}
// 查找当前正在使用的制品
message FindPublicIPLibraryArtifactRequest {
}
message FindPublicIPLibraryArtifactResponse {
IPLibraryArtifact ipLibraryArtifact = 1;
}
// 删除制品
message DeleteIPLibraryArtifactRequest {
int64 ipLibraryArtifactId = 1;
}