优化IP库生成文件的尺寸

build/rpc.json

@@ -19153,7 +19153,7 @@
     },
     {
       "name": "IPLibraryArtifact",
-      "code": "message IPLibraryArtifact {\n\tint64 id = 1;\n\tint64 fileId = 2;\n\tint64 createdAt = 3;\n\tbytes metaJSON = 4;\n\tbool isPublic = 5;\n\tstring name = 6;\n\tstring code = 7;\n}",
+      "code": "message IPLibraryArtifact {\n\tint64 id = 1;\n\tint64 fileId = 2;\n\tint64 createdAt = 3;\n\tbytes metaJSON = 4;\n\tbool isPublic = 5; // 是否公开\n\tstring name = 6;\n\tstring code = 7;\n\n\tFile file = 30; // 文件信息\n}",
       "doc": ""
     },
     {

pkg/iplibrary/.gitignore

@@ -1 +1 @@
-ip.db
+*-plus.db

pkg/iplibrary/reader.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 )
 
+// Reader IP库Reader
 type Reader struct {
 	meta *Meta
 
@@ -21,8 +22,16 @@ type Reader struct {
 
 	ipV4Items []*ipItem
 	ipV6Items []*ipItem
+
+	lastIPFrom     uint64
+	lastCountryId  uint32
+	lastProvinceId uint32
+	lastCityId     uint32
+	lastTownId     uint32
+	lastProviderId uint32
 }
 
+// NewReader 创建新Reader对象
 func NewReader(reader io.Reader) (*Reader, error) {
 	var libReader = &Reader{
 		regionMap: map[string]*ipRegion{},
@@ -34,6 +43,7 @@ func NewReader(reader io.Reader) (*Reader, error) {
 	return libReader, nil
 }
 
+// 从Reader中加载数据
 func (this *Reader) load(reader io.Reader) error {
 	var buf = make([]byte, 1024)
 	var metaLine = []byte{}
@@ -157,17 +167,40 @@ func (this *Reader) IPv6Items() []*ipItem {
 	return this.ipV6Items
 }
 
+// 分析数据
 func (this *Reader) parse(data []byte) (left []byte, err error) {
 	if len(data) == 0 {
 		return
 	}
+
 	for {
 		var index = bytes.IndexByte(data, '\n')
 		if index >= 0 {
 			var line = data[:index]
+			err = this.parseLine(line)
+			if err != nil {
+				return nil, err
+			}
+			data = data[index+1:]
+		} else {
+			left = data
+			break
+		}
+	}
+	return
+}
+
+// 单行分析
+func (this *Reader) parseLine(line []byte) error {
+	const maxPieces = 8
 	var pieces = strings.Split(string(line), "|")
-			if len(pieces) != 8 {
-				return nil, errors.New("invalid ip definition '" + string(line) + "'")
+	var countPieces = len(pieces)
+	if countPieces < maxPieces { // 补足一行
+		for i := 0; i < maxPieces-countPieces; i++ {
+			pieces = append(pieces, "")
+		}
+	} else if countPieces > maxPieces {
+		return errors.New("invalid ip definition '" + string(line) + "'")
 	}
 
 	var version = pieces[0]
@@ -176,25 +209,68 @@ func (this *Reader) parse(data []byte) (left []byte, err error) {
 	}
 
 	if version != "4" && version != "6" {
-				return nil, errors.New("invalid ip version '" + string(line) + "'")
+		return errors.New("invalid ip version '" + string(line) + "'")
 	}
 
+	// ip range
 	var ipFrom uint64
 	var ipTo uint64
-			if len(pieces[2]) == 0 {
-				pieces[2] = pieces[1]
-				ipFrom = types.Uint64(pieces[1])
-				ipTo = types.Uint64(pieces[2])
+	if strings.HasPrefix(pieces[1], "+") {
+		ipFrom = this.lastIPFrom + types.Uint64(pieces[1][1:])
 	} else {
 		ipFrom = types.Uint64(pieces[1])
+	}
+	if len(pieces[2]) == 0 {
+		ipTo = ipFrom
+	} else {
 		ipTo = types.Uint64(pieces[2]) + ipFrom
 	}
+	this.lastIPFrom = ipFrom
+
+	// country
+	var countryId uint32
+	if pieces[3] == "+" {
+		countryId = this.lastCountryId
+	} else {
+		countryId = types.Uint32(pieces[3])
+	}
+	this.lastCountryId = countryId
+
+	var provinceId uint32
+	if pieces[4] == "+" {
+		provinceId = this.lastProvinceId
+	} else {
+		provinceId = types.Uint32(pieces[4])
+	}
+	this.lastProvinceId = provinceId
+
+	// city
+	var cityId uint32
+	if pieces[5] == "+" {
+		cityId = this.lastCityId
+	} else {
+		cityId = types.Uint32(pieces[5])
+	}
+	this.lastCityId = cityId
+
+	// town
+	var townId uint32
+	if pieces[6] == "+" {
+		townId = this.lastTownId
+	} else {
+		townId = types.Uint32(pieces[6])
+	}
+	this.lastTownId = townId
+
+	// provider
+	var providerId uint32
+	if pieces[7] == "+" {
+		providerId = this.lastProviderId
+	} else {
+		providerId = types.Uint32(pieces[7])
+	}
+	this.lastProviderId = providerId
 
-			var countryId = types.Uint32(pieces[3])
-			var provinceId = types.Uint32(pieces[4])
-			var cityId = types.Uint32(pieces[5])
-			var townId = types.Uint32(pieces[6])
-			var providerId = types.Uint32(pieces[7])
 	var hash = HashRegion(countryId, provinceId, cityId, townId, providerId)
 
 	region, ok := this.regionMap[hash]
@@ -223,11 +299,5 @@ func (this *Reader) parse(data []byte) (left []byte, err error) {
 		})
 	}
 
-			data = data[index+1:]
-		} else {
-			left = data
-			break
-		}
-	}
-	return
+	return nil
 }

pkg/iplibrary/writer.go

@@ -41,6 +41,13 @@ func (this *hashWriter) Sum() string {
 type Writer struct {
 	writer *hashWriter
 	meta   *Meta
+
+	lastIPFrom     uint64 // 上一次的IP
+	lastCountryId  int64
+	lastProvinceId int64
+	lastCityId     int64
+	lastTownId     int64
+	lastProviderId int64
 }
 
 func NewWriter(writer io.Writer, meta *Meta) *Writer {
@@ -103,7 +110,12 @@ func (this *Writer) Write(ipFrom string, ipTo string, countryId int64, provinceI
 		fromIPLong, toIPLong = toIPLong, fromIPLong
 	}
 
+	if this.lastIPFrom > 0 && fromIPLong > this.lastIPFrom {
+		pieces = append(pieces, "+"+types.String(fromIPLong-this.lastIPFrom))
+	} else {
 		pieces = append(pieces, types.String(fromIPLong))
+	}
+	this.lastIPFrom = fromIPLong
 	if ipFrom == ipTo {
 		// 2
 		pieces = append(pieces, "")
@@ -114,40 +126,65 @@ func (this *Writer) Write(ipFrom string, ipTo string, countryId int64, provinceI
 
 	// 3
 	if countryId > 0 {
+		if countryId == this.lastCountryId {
+			pieces = append(pieces, "+")
+		} else {
 			pieces = append(pieces, types.String(countryId))
+		}
 	} else {
 		pieces = append(pieces, "")
 	}
+	this.lastCountryId = countryId
 
 	// 4
 	if provinceId > 0 {
+		if provinceId == this.lastProvinceId {
+			pieces = append(pieces, "+")
+		} else {
 			pieces = append(pieces, types.String(provinceId))
+		}
 	} else {
 		pieces = append(pieces, "")
 	}
+	this.lastProvinceId = provinceId
 
 	// 5
 	if cityId > 0 {
+		if cityId == this.lastCityId {
+			pieces = append(pieces, "+")
+		} else {
 			pieces = append(pieces, types.String(cityId))
+		}
 	} else {
 		pieces = append(pieces, "")
 	}
+	this.lastCityId = cityId
 
 	// 6
 	if townId > 0 {
+		if townId == this.lastTownId {
+			pieces = append(pieces, "+")
+		} else {
 			pieces = append(pieces, types.String(townId))
+		}
 	} else {
 		pieces = append(pieces, "")
 	}
+	this.lastTownId = townId
 
 	// 7
 	if providerId > 0 {
+		if providerId == this.lastProviderId {
+			pieces = append(pieces, "+")
+		} else {
 			pieces = append(pieces, types.String(providerId))
+		}
 	} else {
 		pieces = append(pieces, "")
 	}
+	this.lastProviderId = providerId
 
-	_, err := this.writer.Write([]byte(strings.Join(pieces, "|")))
+	_, err := this.writer.Write([]byte(strings.TrimRight(strings.Join(pieces, "|"), "|")))
 	if err != nil {
 		return err
 	}