优化代码/支持IP名单的更多格式的导入、导出

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-yaml/yaml v2.1.0+incompatible
-	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/iwind/TeaGo v0.0.0-20210720011303-fc255c995afa
 	github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
@@ -21,5 +20,6 @@ require (
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
 	google.golang.org/grpc v1.38.0
+	google.golang.org/protobuf v1.26.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )

internal/web/actions/default/admins/recipients/index.go

@@ -39,6 +39,10 @@ func (this *IndexAction) RunGet(params struct {
 		Offset:                  page.Offset,
 		Size:                    page.Size,
 	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
 
 	recipientMaps := []maps.Map{}
 	for _, recipient := range recipientsResp.MessageRecipients {

internal/web/actions/default/admins/recipients/instances/index.go

@@ -35,6 +35,10 @@ func (this *IndexAction) RunGet(params struct {
 		Offset:    page.Offset,
 		Size:      page.Size,
 	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
 
 	instanceMaps := []maps.Map{}
 	for _, instance := range instancesResp.MessageMediaInstances {

internal/web/actions/default/clusters/cluster/node/logs.go

@@ -62,6 +62,10 @@ func (this *LogsAction) RunGet(params struct {
 		Offset:  page.Offset,
 		Size:    page.Size,
 	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
 
 	logs := []maps.Map{}
 	for _, log := range logsResp.NodeLogs {

internal/web/actions/default/clusters/logs/index.go

@@ -51,6 +51,10 @@ func (this *IndexAction) RunGet(params struct {
 		Offset:  page.Offset,
 		Size:    page.Size,
 	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
 
 	logs := []maps.Map{}
 	for _, log := range logsResp.NodeLogs {

internal/web/actions/default/servers/iplists/exportData.go

@@ -3,10 +3,15 @@
 package iplists
 
 import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeAdmin/internal/utils/numberutils"
 	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/golang/protobuf/proto"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	"github.com/tealeg/xlsx/v3"
 	"strconv"
 )
 
@@ -20,10 +25,55 @@ func (this *ExportDataAction) Init() {
 
 func (this *ExportDataAction) RunGet(params struct {
 	ListId int64
+	Format string
 }) {
 	defer this.CreateLogInfo("导出IP名单 %d", params.ListId)
 
-	resp := &pb.ListIPItemsWithListIdResponse{}
+	var err error
+	var ext = ""
+	var jsonMaps = []maps.Map{}
+	var xlsxFile *xlsx.File
+	var xlsxSheet *xlsx.Sheet
+	var csvWriter *csv.Writer
+	var csvBuffer *bytes.Buffer
+
+	var data []byte
+
+	switch params.Format {
+	case "xlsx":
+		ext = ".xlsx"
+		xlsxFile = xlsx.NewFile()
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+		xlsxSheet, err = xlsxFile.AddSheet("IP名单")
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+
+		row := xlsxSheet.AddRow()
+		row.SetHeight(26)
+		row.AddCell().SetValue("开始IP")
+		row.AddCell().SetValue("结束IP")
+		row.AddCell().SetValue("过期时间戳")
+		row.AddCell().SetValue("类型")
+		row.AddCell().SetValue("级别")
+		row.AddCell().SetValue("备注")
+	case "csv":
+		ext = ".csv"
+		csvBuffer = &bytes.Buffer{}
+		csvWriter = csv.NewWriter(csvBuffer)
+	case "txt":
+		ext = ".txt"
+	case "json":
+		ext = ".json"
+	default:
+		this.WriteString("请选择正确的导出格式")
+		return
+	}
+
 	var offset int64 = 0
 	var size int64 = 1000
 	for {
@@ -40,18 +90,60 @@ func (this *ExportDataAction) RunGet(params struct {
 			break
 		}
 		for _, item := range itemsResp.IpItems {
-			resp.IpItems = append(resp.IpItems, item)
+			switch params.Format {
+			case "xlsx":
+				row := xlsxSheet.AddRow()
+				row.SetHeight(26)
+				row.AddCell().SetValue(item.IpFrom)
+				row.AddCell().SetValue(item.IpTo)
+				row.AddCell().SetValue(types.String(item.ExpiredAt))
+				row.AddCell().SetValue(item.Type)
+				row.AddCell().SetValue(item.EventLevel)
+				row.AddCell().SetValue(item.Reason)
+			case "csv":
+				err = csvWriter.Write([]string{item.IpFrom, item.IpTo, types.String(item.ExpiredAt), item.Type, item.EventLevel, item.Reason})
+				if err != nil {
+					this.ErrorPage(err)
+					return
+				}
+			case "txt":
+				data = append(data, item.IpFrom+","+item.IpTo+","+types.String(item.ExpiredAt)+","+item.Type+","+item.EventLevel+","+item.Reason...)
+				data = append(data, '\n')
+			case "json":
+				jsonMaps = append(jsonMaps, maps.Map{
+					"ipFrom":     item.IpFrom,
+					"ipTo":       item.IpTo,
+					"expiredAt":  item.ExpiredAt,
+					"type":       item.Type,
+					"eventLevel": item.EventLevel,
+					"reason":     item.Reason,
+				})
+			}
 		}
 		offset += size
 	}
 
-	data, err := proto.Marshal(resp)
+	switch params.Format {
-	if err != nil {
+	case "xlsx":
-		this.ErrorPage(err)
+		var buf = &bytes.Buffer{}
-		return
+		err = xlsxFile.Write(buf)
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+		data = buf.Bytes()
+	case "csv":
+		csvWriter.Flush()
+		data = csvBuffer.Bytes()
+	case "json":
+		data, err = json.Marshal(jsonMaps)
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
 	}
 
-	this.AddHeader("Content-Disposition", "attachment; filename=\"ip-list-"+numberutils.FormatInt64(params.ListId)+".data\";")
+	this.AddHeader("Content-Disposition", "attachment; filename=\"ip-list-"+numberutils.FormatInt64(params.ListId)+ext+"\";")
 	this.AddHeader("Content-Length", strconv.Itoa(len(data)))
 	this.Write(data)
 }

internal/web/actions/default/servers/iplists/import.go

@@ -3,10 +3,19 @@
 package iplists
 
 import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/golang/protobuf/proto"
 	"github.com/iwind/TeaGo/actions"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/types"
+	"github.com/tealeg/xlsx/v3"
+	"io"
+	"net"
+	"regexp"
+	"strings"
 )
 
 type ImportAction struct {
@@ -51,20 +60,92 @@ func (this *ImportAction) RunPost(params struct {
 		this.Fail("请选择要导入的IP文件")
 	}
 
+	// 检查文件扩展名
+	if !regexp.MustCompile(`(?i)\.(xlsx|csv|json|txt)$`).MatchString(params.File.Filename) {
+		this.Fail("不支持当前格式的文件导入")
+	}
+
+	var ext = strings.ToLower(params.File.Ext)
+
 	data, err := params.File.Read()
 	if err != nil {
 		this.ErrorPage(err)
 		return
 	}
-	resp := &pb.ListIPItemsWithListIdResponse{}
+
-	err = proto.Unmarshal(data, resp)
+	var countIgnore = 0
-	if err != nil {
+	var items = []*pb.IPItem{}
-		this.Fail("导入失败，文件格式错误：" + err.Error())
+	switch ext {
+	case ".xlsx":
+		file, err := xlsx.OpenBinary(data)
+		if err != nil {
+			this.Fail("Excel读取错误：" + err.Error())
+		}
+		if len(file.Sheets) > 0 {
+			var sheet = file.Sheets[0]
+			err = sheet.ForEachRow(func(r *xlsx.Row) error {
+				var values = []string{}
+				err = r.ForEachCell(func(c *xlsx.Cell) error {
+					values = append(values, c.Value)
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+				if len(values) == 0 {
+					return nil
+				}
+				if values[0] == "开始IP" || values[0] == "IP" {
+					return nil
+				}
+				item := this.createItemFromValues(values, &countIgnore)
+				if item != nil {
+					items = append(items, item)
+				}
+				return nil
+			})
+			if err != nil {
+				this.Fail("Excel读取错误：" + err.Error())
+			}
+		}
+	case ".csv":
+		reader := csv.NewReader(bytes.NewBuffer(data))
+		for {
+			values, err := reader.Read()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				this.Fail("CSV读取错误：" + err.Error())
+			}
+			item := this.createItemFromValues(values, &countIgnore)
+			if item != nil {
+				items = append(items, item)
+			}
+		}
+	case ".json":
+		err = json.Unmarshal(data, &items)
+		if err != nil {
+			this.Fail("导入失败：" + err.Error())
+		}
+	case ".txt":
+		lines := bytes.Split(data, []byte{'\n'})
+		for _, line := range lines {
+			if len(line) == 0 {
+				continue
+			}
+			item := this.createItemFromValues(strings.SplitN(string(line), ",", 6), &countIgnore)
+			if item != nil {
+				items = append(items, item)
+			}
+		}
 	}
 
 	var count = 0
-	var countIgnore = 0
+
-	for _, item := range resp.IpItems {
+	lists.Reverse(items)
+
+	for _, item := range items {
 		_, err = this.RPC().IPItemRPC().CreateIPItem(this.AdminContext(), &pb.CreateIPItemRequest{
 			IpListId:   params.ListId,
 			IpFrom:     item.IpFrom,
@@ -85,3 +166,48 @@ func (this *ImportAction) RunPost(params struct {
 
 	this.Success()
 }
+
+func (this *ImportAction) createItemFromValues(values []string, countIgnore *int) *pb.IPItem {
+	// ipFrom, ipTo, expiredAt, type, eventType, reason
+
+	var item = &pb.IPItem{}
+	switch len(values) {
+	case 1:
+		item.IpFrom = values[0]
+	case 2:
+		item.IpFrom = values[0]
+		item.IpTo = values[1]
+	case 3:
+		item.IpFrom = values[0]
+		item.IpTo = values[1]
+		item.ExpiredAt = types.Int64(values[2])
+	case 4:
+		item.IpFrom = values[0]
+		item.IpTo = values[1]
+		item.ExpiredAt = types.Int64(values[2])
+		item.Type = values[3]
+	case 5:
+		item.IpFrom = values[0]
+		item.IpTo = values[1]
+		item.ExpiredAt = types.Int64(values[2])
+		item.Type = values[3]
+		item.EventLevel = values[4]
+	case 6:
+		item.IpFrom = values[0]
+		item.IpTo = values[1]
+		item.ExpiredAt = types.Int64(values[2])
+		item.Type = values[3]
+		item.EventLevel = values[4]
+		item.Reason = values[5]
+	}
+
+	if net.ParseIP(item.IpFrom) == nil {
+		*countIgnore++
+		return nil
+	}
+	if len(item.IpTo) > 0 && net.ParseIP(item.IpTo) == nil {
+		*countIgnore++
+		return nil
+	}
+	return item
+}

internal/web/actions/default/servers/server/settings/locations/sort.go

@@ -27,6 +27,7 @@ func (this *SortAction) RunPost(params struct {
 	}
 	if webConfig == nil {
 		this.Success()
+		return
 	}
 	refMap := map[int64]*serverconfigs.HTTPLocationRef{}
 	for _, ref := range webConfig.LocationRefs {

internal/web/actions/default/servers/server/settings/reverseProxy/setting.go

@@ -121,6 +121,10 @@ func (this *SettingAction) RunPost(params struct {
 		MaxConns:        types.Int32(reverseProxyConfig.MaxConns),
 		MaxIdleConns:    types.Int32(reverseProxyConfig.MaxIdleConns),
 	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
 
 	this.Success()
 }

web/views/@default/servers/iplists/export.html

@@ -2,12 +2,24 @@
 {$template "list_menu"}
 
 <div class="margin"></div>
-<form class="ui form">
+<form class="ui form" method="get" action="/servers/iplists/exportData">
+    <input type="hidden" name="listId" :value="list.id"/>
     <table class="ui table definition selectable">
         <tr>
             <td class="title">说明</td>
             <td>导出所有的IP</td>
         </tr>
+        <tr>
+            <td>格式</td>
+            <td>
+                <select class="ui dropdown auto-width" name="format">
+                    <option value="xlsx">Excel</option>
+                    <option value="csv">CSV</option>
+                    <option value="txt">TXT</option>
+                    <option value="json">JSON</option>
+                </select>
+            </td>
+        </tr>
     </table>
-    <a :href="'/servers/iplists/exportData?listId=' + list.id" class="ui button primary">导出</a>
+    <submit-btn>导出</submit-btn>
 </form>

web/views/@default/servers/iplists/import.html

@@ -3,15 +3,15 @@
 
 <div class="margin"></div>
 
-<form class="ui form" data-tea-action="$"  data-tea-success="success">
+<form class="ui form" data-tea-action="$"  data-tea-success="success" data-tea-timeout="300">
     <input type="hidden" name="listId" :value="list.id"/>
     <csrf-token></csrf-token>
     <table class="ui table definition selectable">
         <tr>
             <td class="title">选择IP文件 *</td>
             <td>
-                <input type="file" name="file" accept=".data"/>
+                <input type="file" name="file" accept=".xlsx, .json, .txt, .csv"/>
-                <p class="comment">文件名类似于<code-label>ip-list-123.data</code-label>。</p>
+                <p class="comment">文件名类似于<code-label>ip-list-123.xxx</code-label>，支持Excel、CSV、JSON和纯文本TXT。</p>
             </td>
         </tr>
     </table>