diff --git a/build/rpc.json b/build/rpc.json index b1bb556..4b63aaa 100644 --- a/build/rpc.json +++ b/build/rpc.json @@ -19153,7 +19153,7 @@ }, { "name": "IPLibraryArtifact", - "code": "message IPLibraryArtifact {\n\tint64 id = 1;\n\tint64 fileId = 2;\n\tint64 createdAt = 3;\n\tbytes metaJSON = 4;\n\tbool isPublic = 5;\n\tstring name = 6;\n\tstring code = 7;\n}", + "code": "message IPLibraryArtifact {\n\tint64 id = 1;\n\tint64 fileId = 2;\n\tint64 createdAt = 3;\n\tbytes metaJSON = 4;\n\tbool isPublic = 5; // 是否公开\n\tstring name = 6;\n\tstring code = 7;\n\n\tFile file = 30; // 文件信息\n}", "doc": "" }, { diff --git a/pkg/iplibrary/.gitignore b/pkg/iplibrary/.gitignore index 0373235..c657966 100644 --- a/pkg/iplibrary/.gitignore +++ b/pkg/iplibrary/.gitignore @@ -1 +1 @@ -ip.db \ No newline at end of file +*-plus.db \ No newline at end of file diff --git a/pkg/iplibrary/reader.go b/pkg/iplibrary/reader.go index 8933a53..ed83f3e 100644 --- a/pkg/iplibrary/reader.go +++ b/pkg/iplibrary/reader.go @@ -14,6 +14,7 @@ import ( "strings" ) +// Reader IP库Reader type Reader struct { meta *Meta @@ -21,8 +22,16 @@ type Reader struct { ipV4Items []*ipItem ipV6Items []*ipItem + + lastIPFrom uint64 + lastCountryId uint32 + lastProvinceId uint32 + lastCityId uint32 + lastTownId uint32 + lastProviderId uint32 } +// NewReader 创建新Reader对象 func NewReader(reader io.Reader) (*Reader, error) { var libReader = &Reader{ regionMap: map[string]*ipRegion{}, @@ -34,6 +43,7 @@ func NewReader(reader io.Reader) (*Reader, error) { return libReader, nil } +// 从Reader中加载数据 func (this *Reader) load(reader io.Reader) error { var buf = make([]byte, 1024) var metaLine = []byte{} @@ -157,72 +167,20 @@ func (this *Reader) IPv6Items() []*ipItem { return this.ipV6Items } +// 分析数据 func (this *Reader) parse(data []byte) (left []byte, err error) { if len(data) == 0 { return } + for { var index = bytes.IndexByte(data, '\n') if index >= 0 { var line = data[:index] - var pieces = strings.Split(string(line), "|") - if len(pieces) != 8 { - return nil, errors.New("invalid ip definition '" + string(line) + "'") + err = this.parseLine(line) + if err != nil { + return nil, err } - - var version = pieces[0] - if len(version) == 0 { - version = "4" - } - - if version != "4" && version != "6" { - return nil, errors.New("invalid ip version '" + string(line) + "'") - } - - var ipFrom uint64 - var ipTo uint64 - if len(pieces[2]) == 0 { - pieces[2] = pieces[1] - ipFrom = types.Uint64(pieces[1]) - ipTo = types.Uint64(pieces[2]) - } else { - ipFrom = types.Uint64(pieces[1]) - ipTo = types.Uint64(pieces[2]) + ipFrom - } - - var countryId = types.Uint32(pieces[3]) - var provinceId = types.Uint32(pieces[4]) - var cityId = types.Uint32(pieces[5]) - var townId = types.Uint32(pieces[6]) - var providerId = types.Uint32(pieces[7]) - var hash = HashRegion(countryId, provinceId, cityId, townId, providerId) - - region, ok := this.regionMap[hash] - if !ok { - region = &ipRegion{ - CountryId: countryId, - ProvinceId: provinceId, - CityId: cityId, - TownId: townId, - ProviderId: providerId, - } - this.regionMap[hash] = region - } - - if version == "4" { - this.ipV4Items = append(this.ipV4Items, &ipItem{ - IPFrom: ipFrom, - IPTo: ipTo, - Region: region, - }) - } else { - this.ipV6Items = append(this.ipV6Items, &ipItem{ - IPFrom: ipFrom, - IPTo: ipTo, - Region: region, - }) - } - data = data[index+1:] } else { left = data @@ -231,3 +189,115 @@ func (this *Reader) parse(data []byte) (left []byte, err error) { } return } + +// 单行分析 +func (this *Reader) parseLine(line []byte) error { + const maxPieces = 8 + var pieces = strings.Split(string(line), "|") + var countPieces = len(pieces) + if countPieces < maxPieces { // 补足一行 + for i := 0; i < maxPieces-countPieces; i++ { + pieces = append(pieces, "") + } + } else if countPieces > maxPieces { + return errors.New("invalid ip definition '" + string(line) + "'") + } + + var version = pieces[0] + if len(version) == 0 { + version = "4" + } + + if version != "4" && version != "6" { + return errors.New("invalid ip version '" + string(line) + "'") + } + + // ip range + var ipFrom uint64 + var ipTo uint64 + if strings.HasPrefix(pieces[1], "+") { + ipFrom = this.lastIPFrom + types.Uint64(pieces[1][1:]) + } else { + ipFrom = types.Uint64(pieces[1]) + } + if len(pieces[2]) == 0 { + ipTo = ipFrom + } else { + ipTo = types.Uint64(pieces[2]) + ipFrom + } + this.lastIPFrom = ipFrom + + // country + var countryId uint32 + if pieces[3] == "+" { + countryId = this.lastCountryId + } else { + countryId = types.Uint32(pieces[3]) + } + this.lastCountryId = countryId + + var provinceId uint32 + if pieces[4] == "+" { + provinceId = this.lastProvinceId + } else { + provinceId = types.Uint32(pieces[4]) + } + this.lastProvinceId = provinceId + + // city + var cityId uint32 + if pieces[5] == "+" { + cityId = this.lastCityId + } else { + cityId = types.Uint32(pieces[5]) + } + this.lastCityId = cityId + + // town + var townId uint32 + if pieces[6] == "+" { + townId = this.lastTownId + } else { + townId = types.Uint32(pieces[6]) + } + this.lastTownId = townId + + // provider + var providerId uint32 + if pieces[7] == "+" { + providerId = this.lastProviderId + } else { + providerId = types.Uint32(pieces[7]) + } + this.lastProviderId = providerId + + var hash = HashRegion(countryId, provinceId, cityId, townId, providerId) + + region, ok := this.regionMap[hash] + if !ok { + region = &ipRegion{ + CountryId: countryId, + ProvinceId: provinceId, + CityId: cityId, + TownId: townId, + ProviderId: providerId, + } + this.regionMap[hash] = region + } + + if version == "4" { + this.ipV4Items = append(this.ipV4Items, &ipItem{ + IPFrom: ipFrom, + IPTo: ipTo, + Region: region, + }) + } else { + this.ipV6Items = append(this.ipV6Items, &ipItem{ + IPFrom: ipFrom, + IPTo: ipTo, + Region: region, + }) + } + + return nil +} diff --git a/pkg/iplibrary/writer.go b/pkg/iplibrary/writer.go index f5fa234..4310621 100644 --- a/pkg/iplibrary/writer.go +++ b/pkg/iplibrary/writer.go @@ -41,6 +41,13 @@ func (this *hashWriter) Sum() string { type Writer struct { writer *hashWriter meta *Meta + + lastIPFrom uint64 // 上一次的IP + lastCountryId int64 + lastProvinceId int64 + lastCityId int64 + lastTownId int64 + lastProviderId int64 } func NewWriter(writer io.Writer, meta *Meta) *Writer { @@ -103,7 +110,12 @@ func (this *Writer) Write(ipFrom string, ipTo string, countryId int64, provinceI fromIPLong, toIPLong = toIPLong, fromIPLong } - pieces = append(pieces, types.String(fromIPLong)) + if this.lastIPFrom > 0 && fromIPLong > this.lastIPFrom { + pieces = append(pieces, "+"+types.String(fromIPLong-this.lastIPFrom)) + } else { + pieces = append(pieces, types.String(fromIPLong)) + } + this.lastIPFrom = fromIPLong if ipFrom == ipTo { // 2 pieces = append(pieces, "") @@ -114,40 +126,65 @@ func (this *Writer) Write(ipFrom string, ipTo string, countryId int64, provinceI // 3 if countryId > 0 { - pieces = append(pieces, types.String(countryId)) + if countryId == this.lastCountryId { + pieces = append(pieces, "+") + } else { + pieces = append(pieces, types.String(countryId)) + } } else { pieces = append(pieces, "") } + this.lastCountryId = countryId // 4 if provinceId > 0 { - pieces = append(pieces, types.String(provinceId)) + if provinceId == this.lastProvinceId { + pieces = append(pieces, "+") + } else { + pieces = append(pieces, types.String(provinceId)) + } } else { pieces = append(pieces, "") } + this.lastProvinceId = provinceId // 5 if cityId > 0 { - pieces = append(pieces, types.String(cityId)) + if cityId == this.lastCityId { + pieces = append(pieces, "+") + } else { + pieces = append(pieces, types.String(cityId)) + } } else { pieces = append(pieces, "") } + this.lastCityId = cityId // 6 if townId > 0 { - pieces = append(pieces, types.String(townId)) + if townId == this.lastTownId { + pieces = append(pieces, "+") + } else { + pieces = append(pieces, types.String(townId)) + } } else { pieces = append(pieces, "") } + this.lastTownId = townId // 7 if providerId > 0 { - pieces = append(pieces, types.String(providerId)) + if providerId == this.lastProviderId { + pieces = append(pieces, "+") + } else { + pieces = append(pieces, types.String(providerId)) + } } else { pieces = append(pieces, "") } + this.lastProviderId = providerId - _, err := this.writer.Write([]byte(strings.Join(pieces, "|"))) + _, err := this.writer.Write([]byte(strings.TrimRight(strings.Join(pieces, "|"), "|"))) if err != nil { return err }