From c3e68915a32945a18ae7260f0e98f797067d26f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Sat, 7 Aug 2021 18:26:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81/=E6=94=AF?= =?UTF-8?q?=E6=8C=81IP=E5=90=8D=E5=8D=95=E7=9A=84=E6=9B=B4=E5=A4=9A?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E7=9A=84=E5=AF=BC=E5=85=A5=E3=80=81=E5=AF=BC?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- .../default/admins/recipients/index.go | 4 + .../admins/recipients/instances/index.go | 4 + .../default/clusters/cluster/node/logs.go | 4 + .../actions/default/clusters/logs/index.go | 4 + .../default/servers/iplists/exportData.go | 108 +++++++++++++- .../actions/default/servers/iplists/import.go | 140 +++++++++++++++++- .../servers/server/settings/locations/sort.go | 1 + .../server/settings/reverseProxy/setting.go | 4 + .../@default/servers/iplists/export.html | 16 +- .../@default/servers/iplists/import.html | 6 +- 11 files changed, 272 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index bb801849..b521ac09 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/go-sql-driver/mysql v1.5.0 github.com/go-yaml/yaml v2.1.0+incompatible - github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.6 // indirect github.com/iwind/TeaGo v0.0.0-20210720011303-fc255c995afa github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 @@ -21,5 +20,6 @@ require ( golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect google.golang.org/grpc v1.38.0 + google.golang.org/protobuf v1.26.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/internal/web/actions/default/admins/recipients/index.go b/internal/web/actions/default/admins/recipients/index.go index a0d576f8..39c6b106 100644 --- a/internal/web/actions/default/admins/recipients/index.go +++ b/internal/web/actions/default/admins/recipients/index.go @@ -39,6 +39,10 @@ func (this *IndexAction) RunGet(params struct { Offset: page.Offset, Size: page.Size, }) + if err != nil { + this.ErrorPage(err) + return + } recipientMaps := []maps.Map{} for _, recipient := range recipientsResp.MessageRecipients { diff --git a/internal/web/actions/default/admins/recipients/instances/index.go b/internal/web/actions/default/admins/recipients/instances/index.go index daf1e1ef..604f9632 100644 --- a/internal/web/actions/default/admins/recipients/instances/index.go +++ b/internal/web/actions/default/admins/recipients/instances/index.go @@ -35,6 +35,10 @@ func (this *IndexAction) RunGet(params struct { Offset: page.Offset, Size: page.Size, }) + if err != nil { + this.ErrorPage(err) + return + } instanceMaps := []maps.Map{} for _, instance := range instancesResp.MessageMediaInstances { diff --git a/internal/web/actions/default/clusters/cluster/node/logs.go b/internal/web/actions/default/clusters/cluster/node/logs.go index aa6b28de..205e7072 100644 --- a/internal/web/actions/default/clusters/cluster/node/logs.go +++ b/internal/web/actions/default/clusters/cluster/node/logs.go @@ -62,6 +62,10 @@ func (this *LogsAction) RunGet(params struct { Offset: page.Offset, Size: page.Size, }) + if err != nil { + this.ErrorPage(err) + return + } logs := []maps.Map{} for _, log := range logsResp.NodeLogs { diff --git a/internal/web/actions/default/clusters/logs/index.go b/internal/web/actions/default/clusters/logs/index.go index 41f0be3d..4b455f44 100644 --- a/internal/web/actions/default/clusters/logs/index.go +++ b/internal/web/actions/default/clusters/logs/index.go @@ -51,6 +51,10 @@ func (this *IndexAction) RunGet(params struct { Offset: page.Offset, Size: page.Size, }) + if err != nil { + this.ErrorPage(err) + return + } logs := []maps.Map{} for _, log := range logsResp.NodeLogs { diff --git a/internal/web/actions/default/servers/iplists/exportData.go b/internal/web/actions/default/servers/iplists/exportData.go index d0159877..a8000323 100644 --- a/internal/web/actions/default/servers/iplists/exportData.go +++ b/internal/web/actions/default/servers/iplists/exportData.go @@ -3,10 +3,15 @@ package iplists import ( + "bytes" + "encoding/csv" + "encoding/json" "github.com/TeaOSLab/EdgeAdmin/internal/utils/numberutils" "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/golang/protobuf/proto" + "github.com/iwind/TeaGo/maps" + "github.com/iwind/TeaGo/types" + "github.com/tealeg/xlsx/v3" "strconv" ) @@ -20,10 +25,55 @@ func (this *ExportDataAction) Init() { func (this *ExportDataAction) RunGet(params struct { ListId int64 + Format string }) { defer this.CreateLogInfo("导出IP名单 %d", params.ListId) - resp := &pb.ListIPItemsWithListIdResponse{} + var err error + var ext = "" + var jsonMaps = []maps.Map{} + var xlsxFile *xlsx.File + var xlsxSheet *xlsx.Sheet + var csvWriter *csv.Writer + var csvBuffer *bytes.Buffer + + var data []byte + + switch params.Format { + case "xlsx": + ext = ".xlsx" + xlsxFile = xlsx.NewFile() + if err != nil { + this.ErrorPage(err) + return + } + xlsxSheet, err = xlsxFile.AddSheet("IP名单") + if err != nil { + this.ErrorPage(err) + return + } + + row := xlsxSheet.AddRow() + row.SetHeight(26) + row.AddCell().SetValue("开始IP") + row.AddCell().SetValue("结束IP") + row.AddCell().SetValue("过期时间戳") + row.AddCell().SetValue("类型") + row.AddCell().SetValue("级别") + row.AddCell().SetValue("备注") + case "csv": + ext = ".csv" + csvBuffer = &bytes.Buffer{} + csvWriter = csv.NewWriter(csvBuffer) + case "txt": + ext = ".txt" + case "json": + ext = ".json" + default: + this.WriteString("请选择正确的导出格式") + return + } + var offset int64 = 0 var size int64 = 1000 for { @@ -40,18 +90,60 @@ func (this *ExportDataAction) RunGet(params struct { break } for _, item := range itemsResp.IpItems { - resp.IpItems = append(resp.IpItems, item) + switch params.Format { + case "xlsx": + row := xlsxSheet.AddRow() + row.SetHeight(26) + row.AddCell().SetValue(item.IpFrom) + row.AddCell().SetValue(item.IpTo) + row.AddCell().SetValue(types.String(item.ExpiredAt)) + row.AddCell().SetValue(item.Type) + row.AddCell().SetValue(item.EventLevel) + row.AddCell().SetValue(item.Reason) + case "csv": + err = csvWriter.Write([]string{item.IpFrom, item.IpTo, types.String(item.ExpiredAt), item.Type, item.EventLevel, item.Reason}) + if err != nil { + this.ErrorPage(err) + return + } + case "txt": + data = append(data, item.IpFrom+","+item.IpTo+","+types.String(item.ExpiredAt)+","+item.Type+","+item.EventLevel+","+item.Reason...) + data = append(data, '\n') + case "json": + jsonMaps = append(jsonMaps, maps.Map{ + "ipFrom": item.IpFrom, + "ipTo": item.IpTo, + "expiredAt": item.ExpiredAt, + "type": item.Type, + "eventLevel": item.EventLevel, + "reason": item.Reason, + }) + } } offset += size } - data, err := proto.Marshal(resp) - if err != nil { - this.ErrorPage(err) - return + switch params.Format { + case "xlsx": + var buf = &bytes.Buffer{} + err = xlsxFile.Write(buf) + if err != nil { + this.ErrorPage(err) + return + } + data = buf.Bytes() + case "csv": + csvWriter.Flush() + data = csvBuffer.Bytes() + case "json": + data, err = json.Marshal(jsonMaps) + if err != nil { + this.ErrorPage(err) + return + } } - this.AddHeader("Content-Disposition", "attachment; filename=\"ip-list-"+numberutils.FormatInt64(params.ListId)+".data\";") + this.AddHeader("Content-Disposition", "attachment; filename=\"ip-list-"+numberutils.FormatInt64(params.ListId)+ext+"\";") this.AddHeader("Content-Length", strconv.Itoa(len(data))) this.Write(data) } diff --git a/internal/web/actions/default/servers/iplists/import.go b/internal/web/actions/default/servers/iplists/import.go index 930114fa..23ef2d59 100644 --- a/internal/web/actions/default/servers/iplists/import.go +++ b/internal/web/actions/default/servers/iplists/import.go @@ -3,10 +3,19 @@ package iplists import ( + "bytes" + "encoding/csv" + "encoding/json" "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/golang/protobuf/proto" "github.com/iwind/TeaGo/actions" + "github.com/iwind/TeaGo/lists" + "github.com/iwind/TeaGo/types" + "github.com/tealeg/xlsx/v3" + "io" + "net" + "regexp" + "strings" ) type ImportAction struct { @@ -51,20 +60,92 @@ func (this *ImportAction) RunPost(params struct { this.Fail("请选择要导入的IP文件") } + // 检查文件扩展名 + if !regexp.MustCompile(`(?i)\.(xlsx|csv|json|txt)$`).MatchString(params.File.Filename) { + this.Fail("不支持当前格式的文件导入") + } + + var ext = strings.ToLower(params.File.Ext) + data, err := params.File.Read() if err != nil { this.ErrorPage(err) return } - resp := &pb.ListIPItemsWithListIdResponse{} - err = proto.Unmarshal(data, resp) - if err != nil { - this.Fail("导入失败，文件格式错误：" + err.Error()) + + var countIgnore = 0 + var items = []*pb.IPItem{} + switch ext { + case ".xlsx": + file, err := xlsx.OpenBinary(data) + if err != nil { + this.Fail("Excel读取错误：" + err.Error()) + } + if len(file.Sheets) > 0 { + var sheet = file.Sheets[0] + err = sheet.ForEachRow(func(r *xlsx.Row) error { + var values = []string{} + err = r.ForEachCell(func(c *xlsx.Cell) error { + values = append(values, c.Value) + return nil + }) + if err != nil { + return err + } + if len(values) == 0 { + return nil + } + if values[0] == "开始IP" || values[0] == "IP" { + return nil + } + item := this.createItemFromValues(values, &countIgnore) + if item != nil { + items = append(items, item) + } + return nil + }) + if err != nil { + this.Fail("Excel读取错误：" + err.Error()) + } + } + case ".csv": + reader := csv.NewReader(bytes.NewBuffer(data)) + for { + values, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + this.Fail("CSV读取错误：" + err.Error()) + } + item := this.createItemFromValues(values, &countIgnore) + if item != nil { + items = append(items, item) + } + } + case ".json": + err = json.Unmarshal(data, &items) + if err != nil { + this.Fail("导入失败：" + err.Error()) + } + case ".txt": + lines := bytes.Split(data, []byte{'\n'}) + for _, line := range lines { + if len(line) == 0 { + continue + } + item := this.createItemFromValues(strings.SplitN(string(line), ",", 6), &countIgnore) + if item != nil { + items = append(items, item) + } + } } var count = 0 - var countIgnore = 0 - for _, item := range resp.IpItems { + + lists.Reverse(items) + + for _, item := range items { _, err = this.RPC().IPItemRPC().CreateIPItem(this.AdminContext(), &pb.CreateIPItemRequest{ IpListId: params.ListId, IpFrom: item.IpFrom, @@ -85,3 +166,48 @@ func (this *ImportAction) RunPost(params struct { this.Success() } + +func (this *ImportAction) createItemFromValues(values []string, countIgnore *int) *pb.IPItem { + // ipFrom, ipTo, expiredAt, type, eventType, reason + + var item = &pb.IPItem{} + switch len(values) { + case 1: + item.IpFrom = values[0] + case 2: + item.IpFrom = values[0] + item.IpTo = values[1] + case 3: + item.IpFrom = values[0] + item.IpTo = values[1] + item.ExpiredAt = types.Int64(values[2]) + case 4: + item.IpFrom = values[0] + item.IpTo = values[1] + item.ExpiredAt = types.Int64(values[2]) + item.Type = values[3] + case 5: + item.IpFrom = values[0] + item.IpTo = values[1] + item.ExpiredAt = types.Int64(values[2]) + item.Type = values[3] + item.EventLevel = values[4] + case 6: + item.IpFrom = values[0] + item.IpTo = values[1] + item.ExpiredAt = types.Int64(values[2]) + item.Type = values[3] + item.EventLevel = values[4] + item.Reason = values[5] + } + + if net.ParseIP(item.IpFrom) == nil { + *countIgnore++ + return nil + } + if len(item.IpTo) > 0 && net.ParseIP(item.IpTo) == nil { + *countIgnore++ + return nil + } + return item +} diff --git a/internal/web/actions/default/servers/server/settings/locations/sort.go b/internal/web/actions/default/servers/server/settings/locations/sort.go index 4ed9dad9..b671678f 100644 --- a/internal/web/actions/default/servers/server/settings/locations/sort.go +++ b/internal/web/actions/default/servers/server/settings/locations/sort.go @@ -27,6 +27,7 @@ func (this *SortAction) RunPost(params struct { } if webConfig == nil { this.Success() + return } refMap := map[int64]*serverconfigs.HTTPLocationRef{} for _, ref := range webConfig.LocationRefs { diff --git a/internal/web/actions/default/servers/server/settings/reverseProxy/setting.go b/internal/web/actions/default/servers/server/settings/reverseProxy/setting.go index 40af41d5..e4bf9f08 100644 --- a/internal/web/actions/default/servers/server/settings/reverseProxy/setting.go +++ b/internal/web/actions/default/servers/server/settings/reverseProxy/setting.go @@ -121,6 +121,10 @@ func (this *SettingAction) RunPost(params struct { MaxConns: types.Int32(reverseProxyConfig.MaxConns), MaxIdleConns: types.Int32(reverseProxyConfig.MaxIdleConns), }) + if err != nil { + this.ErrorPage(err) + return + } this.Success() } diff --git a/web/views/@default/servers/iplists/export.html b/web/views/@default/servers/iplists/export.html index 816fb722..5cc38517 100644 --- a/web/views/@default/servers/iplists/export.html +++ b/web/views/@default/servers/iplists/export.html @@ -2,12 +2,24 @@ {$template "list_menu"}
-
+ + + + + +
说明 导出所有的IP
格式 + +
- 导出 + 导出
\ No newline at end of file diff --git a/web/views/@default/servers/iplists/import.html b/web/views/@default/servers/iplists/import.html index c3b86cc9..41d53e37 100644 --- a/web/views/@default/servers/iplists/import.html +++ b/web/views/@default/servers/iplists/import.html @@ -3,15 +3,15 @@
-
+
选择IP文件 * - -

文件名类似于ip-list-123.data。

+ +

文件名类似于ip-list-123.xxx，支持Excel、CSV、JSON和纯文本TXT。