TRKJ/EdgeCommon
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeCommon.git synced 2025-11-04 05:00:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化IP库内存使用

Browse Source
This commit is contained in:
刘祥超
2022-08-21 23:09:25 +08:00
parent 91b865dd23
commit 49129ea227
7 changed files with 147 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/iplibrary/default_ip_library.go
View File

@@ -6,9 +6,7 @@ import (
"bytes"
"compress/gzip"
_ "embed"
"github.com/iwind/TeaGo/logs"
"net"
"os"
)
//go:embed internal-ip-library.db
@@ -16,16 +14,8 @@ var ipLibraryData []byte
var library = NewIPLibrary()
func init() {
var args = os.Args
if len(args) > 1 && args[1] == "daemon" {
return
}
err := library.Init()
if err != nil {
logs.Println("IP_LIBRARY", "initialized failed: "+err.Error())
}
func Init() error {
return library.Init()
}
func Lookup(ip net.IP) *QueryResult {
@@ -45,6 +35,9 @@ func NewIPLibrary() *IPLibrary {
}
func (this *IPLibrary) Init() error {
if len(ipLibraryData) == 0 || this.reader != nil {
return nil
}
var reader = bytes.NewReader(ipLibraryData)
gzipReader, err := gzip.NewReader(reader)
if err != nil {
@@ -59,6 +52,7 @@ func (this *IPLibrary) Init() error {
return err
}
this.reader = libReader
return nil
}

20
pkg/iplibrary/default_ip_library_test.go
View File

@@ -3,17 +3,37 @@
package iplibrary_test
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"net"
"runtime"
"runtime/debug"
"testing"
"time"
)
func TestIPLibrary_Lookup(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var lib = iplibrary.NewIPLibrary()
var before = time.Now()
err := lib.Init()
if err != nil {
t.Fatal(err)
}
var costMs = time.Since(before).Seconds() * 1000
runtime.GC()
debug.FreeOSMemory()
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log((stat2.Alloc-stat1.Alloc)/1024/1024, "M", fmt.Sprintf("%.2f", costMs), "ms")
for _, ip := range []string{
"127.0.0.1",
"8.8.8.8",

28
pkg/iplibrary/ip_item.go
View File

@@ -2,12 +2,26 @@
package iplibrary
import (
"bytes"
"encoding/binary"
)
type ipItem struct {
ipFrom uint64
ipTo uint64
countryId int64
provinceId int64
cityId int64
townId int64
providerId int64
CountryId uint32
ProvinceId uint32
CityId uint32
TownId uint32
ProviderId uint32
IPFrom uint64
IPTo uint64
}
func (this *ipItem) AsBinary() ([]byte, error) {
var buf = &bytes.Buffer{}
err := binary.Write(buf, binary.BigEndian, this)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}

33
pkg/iplibrary/ip_item_test.go Normal file
View File

@@ -0,0 +1,33 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package iplibrary
import (
"bytes"
"encoding/binary"
"testing"
)
func TestIpItem_AsBinary(t *testing.T) {
var item = &ipItem{
IPFrom: 123456789,
IPTo: 123456790,
CountryId: 1,
ProvinceId: 2,
CityId: 3,
TownId: 4,
ProviderId: 5,
}
b, err := item.AsBinary()
if err != nil {
t.Fatal(err)
}
t.Log(len(b), "bytes")
var item2 = &ipItem{}
err = binary.Read(bytes.NewReader(b), binary.BigEndian, item2)
if err != nil {
t.Fatal(err)
}
t.Log(item2)
}

40
pkg/iplibrary/meta.go
View File

@@ -3,31 +3,31 @@
package iplibrary
type Country struct {
Id int64 `json:"id"`
Id uint32 `json:"id"`
Name string `json:"name"`
Codes []string `json:"codes"`
}
type Province struct {
Id int64 `json:"id"`
Id uint32 `json:"id"`
Name string `json:"name"`
Codes []string `json:"codes"`
}
type City struct {
Id int64 `json:"id"`
Id uint32 `json:"id"`
Name string `json:"name"`
Codes []string `json:"codes"`
}
type Town struct {
Id int64 `json:"id"`
Id uint32 `json:"id"`
Name string `json:"name"`
Codes []string `json:"codes"`
}
type Provider struct {
Id int64 `json:"id"`
Id uint32 `json:"id"`
Name string `json:"name"`
Codes []string `json:"codes"`
}
@@ -42,19 +42,19 @@ type Meta struct {
Providers []*Provider `json:"providers"`
CreatedAt int64 `json:"createdAt"`
countryMap map[int64]*Country // id => *Country
provinceMap map[int64]*Province // id => *Province
cityMap map[int64]*City // id => *City
townMap map[int64]*Town // id => *Town
providerMap map[int64]*Provider // id => *Provider
countryMap map[uint32]*Country // id => *Country
provinceMap map[uint32]*Province // id => *Province
cityMap map[uint32]*City // id => *City
townMap map[uint32]*Town // id => *Town
providerMap map[uint32]*Provider // id => *Provider
}
func (this *Meta) Init() {
this.countryMap = map[int64]*Country{}
this.provinceMap = map[int64]*Province{}
this.cityMap = map[int64]*City{}
this.townMap = map[int64]*Town{}
this.providerMap = map[int64]*Provider{}
this.countryMap = map[uint32]*Country{}
this.provinceMap = map[uint32]*Province{}
this.cityMap = map[uint32]*City{}
this.townMap = map[uint32]*Town{}
this.providerMap = map[uint32]*Provider{}
for _, country := range this.Countries {
this.countryMap[country.Id] = country
@@ -73,22 +73,22 @@ func (this *Meta) Init() {
}
}
func (this *Meta) CountryWithId(countryId int64) *Country {
func (this *Meta) CountryWithId(countryId uint32) *Country {
return this.countryMap[countryId]
}
func (this *Meta) ProvinceWithId(provinceId int64) *Province {
func (this *Meta) ProvinceWithId(provinceId uint32) *Province {
return this.provinceMap[provinceId]
}
func (this *Meta) CityWithId(cityId int64) *City {
func (this *Meta) CityWithId(cityId uint32) *City {
return this.cityMap[cityId]
}
func (this *Meta) TownWithId(townId int64) *Town {
func (this *Meta) TownWithId(townId uint32) *Town {
return this.townMap[townId]
}
func (this *Meta) ProviderWithId(providerId int64) *Provider {
func (this *Meta) ProviderWithId(providerId uint32) *Provider {
return this.providerMap[providerId]
}

52
pkg/iplibrary/reader.go
View File

@@ -77,10 +77,10 @@ func (this *Reader) load(reader io.Reader) error {
}
sort.Slice(this.ipV4Items, func(i, j int) bool {
var from0 = this.ipV4Items[i].ipFrom
var to0 = this.ipV4Items[i].ipTo
var from1 = this.ipV4Items[j].ipFrom
var to1 = this.ipV4Items[j].ipTo
var from0 = this.ipV4Items[i].IPFrom
var to0 = this.ipV4Items[i].IPTo
var from1 = this.ipV4Items[j].IPFrom
var to1 = this.ipV4Items[j].IPTo
if from0 == from1 {
return to0 < to1
}
@@ -88,10 +88,10 @@ func (this *Reader) load(reader io.Reader) error {
})
sort.Slice(this.ipV6Items, func(i, j int) bool {
var from0 = this.ipV6Items[i].ipFrom
var to0 = this.ipV6Items[i].ipTo
var from1 = this.ipV6Items[j].ipFrom
var to1 = this.ipV6Items[j].ipTo
var from0 = this.ipV6Items[i].IPFrom
var to0 = this.ipV6Items[i].IPTo
var from1 = this.ipV6Items[j].IPFrom
var to1 = this.ipV6Items[j].IPTo
if from0 == from1 {
return to0 < to1
}
@@ -112,8 +112,8 @@ func (this *Reader) Lookup(ip net.IP) *QueryResult {
if isV4 {
sort.Search(len(this.ipV4Items), func(i int) bool {
var item = this.ipV4Items[i]
if item.ipFrom <= ipLong {
if item.ipTo >= ipLong {
if item.IPFrom <= ipLong {
if item.IPTo >= ipLong {
resultItem = item
return false
}
@@ -124,8 +124,8 @@ func (this *Reader) Lookup(ip net.IP) *QueryResult {
} else {
sort.Search(len(this.ipV6Items), func(i int) bool {
var item = this.ipV6Items[i]
if item.ipFrom <= ipLong {
if item.ipTo >= ipLong {
if item.IPFrom <= ipLong {
if item.IPTo >= ipLong {
resultItem = item
return false
}
@@ -188,23 +188,23 @@ func (this *Reader) parse(data []byte) (left []byte, err error) {
if version == "4" {
this.ipV4Items = append(this.ipV4Items, &ipItem{
ipFrom: ipFrom,
ipTo: ipTo,
countryId: types.Int64(pieces[3]),
provinceId: types.Int64(pieces[4]),
cityId: types.Int64(pieces[5]),
townId: types.Int64(pieces[6]),
providerId: types.Int64(pieces[7]),
IPFrom: ipFrom,
IPTo: ipTo,
CountryId: types.Uint32(pieces[3]),
ProvinceId: types.Uint32(pieces[4]),
CityId: types.Uint32(pieces[5]),
TownId: types.Uint32(pieces[6]),
ProviderId: types.Uint32(pieces[7]),
})
} else {
this.ipV6Items = append(this.ipV6Items, &ipItem{
ipFrom: ipFrom,
ipTo: ipTo,
countryId: types.Int64(pieces[3]),
provinceId: types.Int64(pieces[4]),
cityId: types.Int64(pieces[5]),
townId: types.Int64(pieces[6]),
providerId: types.Int64(pieces[7]),
IPFrom: ipFrom,
IPTo: ipTo,
CountryId: types.Uint32(pieces[3]),
ProvinceId: types.Uint32(pieces[4]),
CityId: types.Uint32(pieces[5]),
TownId: types.Uint32(pieces[6]),
ProviderId: types.Uint32(pieces[7]),
})
}

42
pkg/iplibrary/reader_result.go
View File

@@ -18,7 +18,7 @@ func (this *QueryResult) IsOk() bool {
func (this *QueryResult) CountryId() int64 {
if this.item != nil {
return this.item.countryId
return int64(this.item.CountryId)
}
return 0
}
@@ -27,8 +27,8 @@ func (this *QueryResult) CountryName() string {
if this.item == nil {
return ""
}
if this.item.countryId > 0 {
var country = this.meta.CountryWithId(this.item.countryId)
if this.item.CountryId > 0 {
var country = this.meta.CountryWithId(this.item.CountryId)
if country != nil {
return country.Name
}
@@ -40,8 +40,8 @@ func (this *QueryResult) CountryCodes() []string {
if this.item == nil {
return nil
}
if this.item.countryId > 0 {
var country = this.meta.CountryWithId(this.item.countryId)
if this.item.CountryId > 0 {
var country = this.meta.CountryWithId(this.item.CountryId)
if country != nil {
return country.Codes
}
@@ -51,7 +51,7 @@ func (this *QueryResult) CountryCodes() []string {
func (this *QueryResult) ProvinceId() int64 {
if this.item != nil {
return this.item.provinceId
return int64(this.item.ProvinceId)
}
return 0
}
@@ -60,8 +60,8 @@ func (this *QueryResult) ProvinceName() string {
if this.item == nil {
return ""
}
if this.item.provinceId > 0 {
var province = this.meta.ProvinceWithId(this.item.provinceId)
if this.item.ProvinceId > 0 {
var province = this.meta.ProvinceWithId(this.item.ProvinceId)
if province != nil {
return province.Name
}
@@ -73,8 +73,8 @@ func (this *QueryResult) ProvinceCodes() []string {
if this.item == nil {
return nil
}
if this.item.provinceId > 0 {
var province = this.meta.ProvinceWithId(this.item.provinceId)
if this.item.ProvinceId > 0 {
var province = this.meta.ProvinceWithId(this.item.ProvinceId)
if province != nil {
return province.Codes
}
@@ -84,7 +84,7 @@ func (this *QueryResult) ProvinceCodes() []string {
func (this *QueryResult) CityId() int64 {
if this.item != nil {
return this.item.cityId
return int64(this.item.CityId)
}
return 0
}
@@ -93,8 +93,8 @@ func (this *QueryResult) CityName() string {
if this.item == nil {
return ""
}
if this.item.cityId > 0 {
var city = this.meta.CityWithId(this.item.cityId)
if this.item.CityId > 0 {
var city = this.meta.CityWithId(this.item.CityId)
if city != nil {
return city.Name
}
@@ -104,7 +104,7 @@ func (this *QueryResult) CityName() string {
func (this *QueryResult) TownId() int64 {
if this.item != nil {
return this.item.townId
return int64(this.item.TownId)
}
return 0
}
@@ -113,8 +113,8 @@ func (this *QueryResult) TownName() string {
if this.item == nil {
return ""
}
if this.item.townId > 0 {
var town = this.meta.TownWithId(this.item.townId)
if this.item.TownId > 0 {
var town = this.meta.TownWithId(this.item.TownId)
if town != nil {
return town.Name
}
@@ -124,7 +124,7 @@ func (this *QueryResult) TownName() string {
func (this *QueryResult) ProviderId() int64 {
if this.item != nil {
return this.item.providerId
return int64(this.item.ProviderId)
}
return 0
}
@@ -133,8 +133,8 @@ func (this *QueryResult) ProviderName() string {
if this.item == nil {
return ""
}
if this.item.providerId > 0 {
var provider = this.meta.ProviderWithId(this.item.providerId)
if this.item.ProviderId > 0 {
var provider = this.meta.ProviderWithId(this.item.ProviderId)
if provider != nil {
return provider.Name
}
@@ -146,8 +146,8 @@ func (this *QueryResult) ProviderCodes() []string {
if this.item == nil {
return nil
}
if this.item.providerId > 0 {
var provider = this.meta.ProviderWithId(this.item.providerId)
if this.item.ProviderId > 0 {
var provider = this.meta.ProviderWithId(this.item.ProviderId)
if provider != nil {
return provider.Codes
}