节点组合配置时服务间可以共用证书数据

build/rpc.json

@@ -16946,7 +16946,7 @@
     },
     {
       "name": "FindCurrentNodeConfigRequest",
-      "code": "message FindCurrentNodeConfigRequest {\n\t// 由于登录信息中已经包含了节点信息，所以这里不需要nodeId\n\tint64 version = 1;\n\tbool compress = 2; // 是否压缩\n\tint64 nodeTaskVersion = 3; // 通知任务版本\n}",
+      "code": "message FindCurrentNodeConfigRequest {\n\t// 由于登录信息中已经包含了节点信息，所以这里不需要nodeId\n\tint64 version = 1;\n\tbool compress = 2; // 是否压缩\n\tint64 nodeTaskVersion = 3; // 通知任务版本\n\tbool useDataMap = 4; // 是否使用公共的数据集\n}",
       "doc": "组合单个节点配置"
     },
     {

pkg/nodeconfigs/node_config.go

@@ -2,6 +2,7 @@ package nodeconfigs
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
@@ -44,6 +45,7 @@ type NodeConfig struct {
 	GroupId      int64                         `yaml:"groupId" json:"groupId"`
 	RegionId     int64                         `yaml:"regionId" json:"regionId"`
 	OCSPVersion  int64                         `yaml:"ocspVersion" json:"ocspVersion"`
+	DataMap      *shared.DataMap               `yaml:"dataMap" json:"dataMap"`
 
 	// 性能
 	MaxCPU       int32                                 `yaml:"maxCPU" json:"maxCPU"`
@@ -209,7 +211,13 @@ func CloneNodeConfig(nodeConfig *NodeConfig) (*NodeConfig, error) {
 }
 
 // Init 初始化
-func (this *NodeConfig) Init() (err error, serverErrors []*ServerError) {
+func (this *NodeConfig) Init(ctx context.Context) (err error, serverErrors []*ServerError) {
+	// 设置Context
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	ctx = context.WithValue(ctx, "DataMap", this.DataMap)
+
 	this.secretHash = fmt.Sprintf("%x", sha256.Sum256([]byte(this.NodeId+"@"+this.Secret)))
 	this.paddedId = fmt.Sprintf("%08d", this.Id)
 
@@ -221,7 +229,7 @@ func (this *NodeConfig) Init() (err error, serverErrors []*ServerError) {
 		}
 
 		// 初始化
-		errs := server.Init()
+		errs := server.Init(ctx)
 		if len(errs) > 0 {
 			// 这里不返回错误，而是继续往下，防止单个服务错误而影响其他服务
 			for _, serverErr := range errs {

pkg/rpc/protos/service_node.proto

@@ -291,6 +291,7 @@ message FindCurrentNodeConfigRequest {
 	int64 version = 1;
 	bool compress = 2; // 是否压缩
 	int64 nodeTaskVersion = 3; // 通知任务版本
+	bool useDataMap = 4; // 是否使用公共的数据集
 }
 
 message FindCurrentNodeConfigResponse {

pkg/serverconfigs/follow_protocol_config.go

@@ -0,0 +1,28 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package serverconfigs
+
+// FollowProtocolConfig 协议跟随配置
+type FollowProtocolConfig struct {
+	IsPrior bool `yaml:"isPrior" json:"isPrior"` // 是否覆盖父级配置
+	IsOn    bool `yaml:"isOn" json:"isOn"`       // 是否启用
+	HTTP    struct {
+		Port       int  `yaml:"port" json:"port"`             // 端口
+		FollowPort bool `yaml:"followPort" json:"followPort"` // 跟随端口
+	} `yaml:"http" json:"http"` // HTTP配置
+	HTTPS struct {
+		Port       int  `yaml:"port" json:"port"`             // 端口
+		FollowPort bool `yaml:"followPort" json:"followPort"` // 跟随端口
+	} `yaml:"https" json:"https"` // HTTPS配置
+}
+
+func NewFollowProtocolConfig() *FollowProtocolConfig {
+	var p = &FollowProtocolConfig{}
+	p.HTTP.FollowPort = true
+	p.HTTPS.FollowPort = true
+	return p
+}
+
+func (this *FollowProtocolConfig) Init() error {
+	return nil
+}

pkg/serverconfigs/http_location_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"regexp"
 	"strconv"
@@ -32,14 +33,14 @@ type HTTPLocationConfig struct {
 	reverse         bool           // 是否翻转规则，比如非前缀，非路径
 }
 
-func (this *HTTPLocationConfig) Init() error {
+func (this *HTTPLocationConfig) Init(ctx context.Context) error {
 	err := this.ExtractPattern()
 	if err != nil {
 		return err
 	}
 
 	if this.Web != nil {
-		err := this.Web.Init()
+		err := this.Web.Init(ctx)
 		if err != nil {
 			return err
 		}
@@ -53,7 +54,7 @@ func (this *HTTPLocationConfig) Init() error {
 	}
 
 	if this.ReverseProxy != nil {
-		err := this.ReverseProxy.Init()
+		err := this.ReverseProxy.Init(ctx)
 		if err != nil {
 			return err
 		}
@@ -61,7 +62,7 @@ func (this *HTTPLocationConfig) Init() error {
 
 	// Children
 	for _, child := range this.Children {
-		err := child.Init()
+		err := child.Init(ctx)
 		if err != nil {
 			return err
 		}

pkg/serverconfigs/http_web_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 )
@@ -54,7 +55,7 @@ type HTTPWebConfig struct {
 	CC  *HTTPCCConfig `yaml:"cc" json:"cc"`
 }
 
-func (this *HTTPWebConfig) Init() error {
+func (this *HTTPWebConfig) Init(ctx context.Context) error {
 	// root
 	if this.Root != nil {
 		err := this.Root.Init()
@@ -66,7 +67,7 @@ func (this *HTTPWebConfig) Init() error {
 	// 路径规则
 	if len(this.Locations) > 0 {
 		for _, location := range this.Locations {
-			err := location.Init()
+			err := location.Init(ctx)
 			if err != nil {
 				return err
 			}

pkg/serverconfigs/origin_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
@@ -30,10 +31,11 @@ type OriginConfig struct {
 
 	Domains []string `yaml:"domains" json:"domains"` // 所属域名
 
-	StripPrefix string `yaml:"stripPrefix" json:"stripPrefix"` // 去除URL前缀
+	StripPrefix    string                `yaml:"stripPrefix" json:"stripPrefix"`       // 去除URL前缀
-	RequestURI  string `yaml:"requestURI" json:"requestURI"`   // 转发后的请求URI TODO
+	RequestURI     string                `yaml:"requestURI" json:"requestURI"`         // 转发后的请求URI TODO
-	RequestHost string `yaml:"requestHost" json:"requestHost"` // 自定义主机名
+	RequestHost    string                `yaml:"requestHost" json:"requestHost"`       // 自定义主机名
-	FollowPort  bool   `yaml:"followPort" json:"followPort"`   // 端口跟随
+	FollowPort     bool                  `yaml:"followPort" json:"followPort"`         // 端口跟随
+	FollowProtocol *FollowProtocolConfig `yaml:"followProtocol" json:"followProtocol"` // 协议跟随 TODO
 
 	RequestHeaderPolicyRef  *shared.HTTPHeaderPolicyRef `yaml:"requestHeaderPolicyRef" json:"requestHeaderPolicyRef"`   // 请求Header
 	RequestHeaderPolicy     *shared.HTTPHeaderPolicy    `yaml:"requestHeaderPolicy" json:"requestHeaderPolicy"`         // 请求Header策略
@@ -71,7 +73,7 @@ type OriginConfig struct {
 }
 
 // Init 校验
-func (this *OriginConfig) Init() error {
+func (this *OriginConfig) Init(ctx context.Context) error {
 	this.IsOk = true
 
 	// URL
@@ -91,7 +93,7 @@ func (this *OriginConfig) Init() error {
 
 	// 证书
 	if this.Cert != nil {
-		err := this.Cert.Init()
+		err := this.Cert.Init(ctx)
 		if err != nil {
 			return err
 		}
@@ -161,6 +163,14 @@ func (this *OriginConfig) Init() error {
 		}
 	}
 
+	// follow protocol
+	if this.FollowProtocol != nil {
+		err := this.FollowProtocol.Init()
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/serverconfigs/origin_config_test.go

@@ -7,7 +7,7 @@ func TestOriginConfig_UniqueKey(t *testing.T) {
 		Id:      1,
 		Version: 101,
 	}
-	err := origin.Init()
+	err := origin.Init(nil)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/serverconfigs/protocol_https_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
 )
@@ -25,14 +26,14 @@ type HTTPSProtocolConfig struct {
 }
 
 // Init 初始化
-func (this *HTTPSProtocolConfig) Init() error {
+func (this *HTTPSProtocolConfig) Init(ctx context.Context) error {
 	err := this.InitBase()
 	if err != nil {
 		return err
 	}
 
 	if this.SSLPolicy != nil {
-		err := this.SSLPolicy.Init()
+		err := this.SSLPolicy.Init(ctx)
 		if err != nil {
 			return err
 		}

pkg/serverconfigs/protocol_tls_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
 )
@@ -25,14 +26,14 @@ type TLSProtocolConfig struct {
 }
 
 // Init 初始化
-func (this *TLSProtocolConfig) Init() error {
+func (this *TLSProtocolConfig) Init(ctx context.Context) error {
 	err := this.InitBase()
 	if err != nil {
 		return err
 	}
 
 	if this.SSLPolicy != nil {
-		err := this.SSLPolicy.Init()
+		err := this.SSLPolicy.Init(ctx)
 		if err != nil {
 			return err
 		}

pkg/serverconfigs/reverse_proxy_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/iwind/TeaGo/lists"
@@ -42,8 +43,9 @@ type ReverseProxyConfig struct {
 
 	AutoFlush bool `yaml:"autoFlush" json:"autoFlush"` // 是否自动刷新缓冲区，在比如SSE（server-sent events）场景下很有用
 
-	ProxyProtocol   *ProxyProtocolConfig `yaml:"proxyProtocol" json:"proxyProtocol"`     // PROXY Protocol
+	ProxyProtocol   *ProxyProtocolConfig  `yaml:"proxyProtocol" json:"proxyProtocol"`     // PROXY Protocol
-	FollowRedirects bool                 `yaml:"followRedirects" json:"followRedirects"` // 回源跟随
+	FollowRedirects bool                  `yaml:"followRedirects" json:"followRedirects"` // 回源跟随
+	FollowProtocol  *FollowProtocolConfig `yaml:"followProtocol" json:"followProtocol"`   // 协议跟随 TODO
 
 	requestHostHasVariables bool
 	requestURIHasVariables  bool
@@ -60,7 +62,7 @@ type ReverseProxyConfig struct {
 }
 
 // Init 初始化
-func (this *ReverseProxyConfig) Init() error {
+func (this *ReverseProxyConfig) Init(ctx context.Context) error {
 	this.requestHostHasVariables = configutils.HasVariables(this.RequestHost)
 	this.requestURIHasVariables = configutils.HasVariables(this.RequestURI)
 
@@ -171,7 +173,7 @@ func (this *ReverseProxyConfig) Init() error {
 			}
 
 			// 初始化
-			err := origin.Init()
+			err := origin.Init(ctx)
 			if err != nil {
 				return err
 			}
@@ -205,6 +207,14 @@ func (this *ReverseProxyConfig) Init() error {
 		}
 	}
 
+	// follow protocol
+	if this.FollowProtocol != nil {
+		err := this.FollowProtocol.Init()
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/serverconfigs/reverse_proxy_config_test.go

@@ -30,7 +30,7 @@ func TestReverseProxyConfig_Init(t *testing.T) {
 		Addr: &NetworkAddressConfig{Host: "127.0.0.4"},
 		IsOn: true,
 	})
-	err := config.Init()
+	err := config.Init(nil)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/serverconfigs/server_config.go

@@ -1,6 +1,7 @@
 package serverconfigs
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
@@ -80,7 +81,7 @@ func NewServerConfig() *ServerConfig {
 	return &ServerConfig{}
 }
 
-func (this *ServerConfig) Init() (results []error) {
+func (this *ServerConfig) Init(ctx context.Context) (results []error) {
 	if this.isInitialized {
 		return
 	}
@@ -193,7 +194,7 @@ func (this *ServerConfig) Init() (results []error) {
 	}
 
 	if this.HTTPS != nil {
-		err := this.HTTPS.Init()
+		err := this.HTTPS.Init(ctx)
 		if err != nil {
 			results = append(results, err)
 		}
@@ -207,7 +208,7 @@ func (this *ServerConfig) Init() (results []error) {
 	}
 
 	if this.TLS != nil {
-		err := this.TLS.Init()
+		err := this.TLS.Init(ctx)
 		if err != nil {
 			results = append(results, err)
 		}
@@ -235,14 +236,14 @@ func (this *ServerConfig) Init() (results []error) {
 	}
 
 	if this.ReverseProxy != nil {
-		err := this.ReverseProxy.Init()
+		err := this.ReverseProxy.Init(ctx)
 		if err != nil {
 			results = append(results, err)
 		}
 	}
 
 	if this.Web != nil {
-		err := this.Web.Init()
+		err := this.Web.Init(ctx)
 		if err != nil {
 			results = append(results, err)
 		}

pkg/serverconfigs/server_config_test.go

@@ -65,7 +65,7 @@ func TestServerConfig_Protocols(t *testing.T) {
 				},
 			},
 		}}
-		err := server.Init()
+		err := server.Init(nil)
 		if err != nil {
 			t.Fatal(err)
 		}

pkg/serverconfigs/shared/data_map.go

@@ -0,0 +1,37 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package shared
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+)
+
+var dataMapPrefix = []byte("GOEDGE_DATA_MAP:")
+
+// DataMap 二进制数据共享Map
+// 用来减少相同数据占用的空间和内存
+type DataMap struct {
+	Map map[string][]byte
+}
+
+// NewDataMap 构建对象
+func NewDataMap() *DataMap {
+	return &DataMap{Map: map[string][]byte{}}
+}
+
+// Put 放入数据
+func (this *DataMap) Put(data []byte) (keyData []byte) {
+	var key = string(dataMapPrefix) + fmt.Sprintf("%x", md5.Sum(data))
+	this.Map[key] = data
+	return []byte(key)
+}
+
+// Read 读取数据
+func (this *DataMap) Read(key []byte) []byte {
+	if bytes.HasPrefix(key, dataMapPrefix) {
+		return this.Map[string(key)]
+	}
+	return key
+}

pkg/serverconfigs/shared/data_map_test.go

@@ -0,0 +1,17 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package shared_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"testing"
+)
+
+func TestNewDataMap(t *testing.T) {
+	var m = shared.NewDataMap()
+	t.Log("data:", m.Read([]byte("e10adc3949ba59abbe56e057f20f883e")))
+	var key = m.Put([]byte("123456"))
+	t.Log("keyData:", key)
+	t.Log("keyString:", string(key))
+	t.Log("data:", string(m.Read(key)))
+}

pkg/serverconfigs/sslconfigs/ssl_cert_config.go

@@ -1,12 +1,15 @@
 package sslconfigs
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/iwind/TeaGo/lists"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -41,7 +44,23 @@ type SSLCertConfig struct {
 }
 
 // Init 校验
-func (this *SSLCertConfig) Init() error {
+func (this *SSLCertConfig) Init(ctx context.Context) error {
+	// 从ctx中读取数据
+	if ctx != nil {
+		var dataMapOne = ctx.Value("DataMap")
+		if dataMapOne != nil && !reflect.ValueOf(dataMapOne).IsNil() {
+			dataMap, ok := dataMapOne.(*shared.DataMap)
+			if !ok {
+				return errors.New("SSLCertConfig.init(): invalid 'DataMap' in context")
+			}
+			if dataMap != nil { // 再次检查是否为nil
+				this.KeyData = dataMap.Read(this.KeyData)
+				this.CertData = dataMap.Read(this.CertData)
+				this.OCSP = dataMap.Read(this.OCSP)
+			}
+		}
+	}
+
 	var commonNames []string // 发行组织
 	var dnsNames []string    // 域名
 

pkg/serverconfigs/sslconfigs/ssl_policy.go

@@ -2,6 +2,7 @@ package sslconfigs
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
@@ -48,13 +49,13 @@ type SSLPolicy struct {
 }
 
 // Init 校验配置
-func (this *SSLPolicy) Init() error {
+func (this *SSLPolicy) Init(ctx context.Context) error {
 	this.nameMapping = map[string]*tls.Certificate{}
 
 	// certs
 	var certs = []tls.Certificate{}
 	for _, cert := range this.Certs {
-		err := cert.Init()
+		err := cert.Init(ctx)
 		if err != nil {
 			return err
 		}
@@ -74,7 +75,7 @@ func (this *SSLPolicy) Init() error {
 	this.clientCAPool = x509.NewCertPool()
 
 	for _, cert := range this.ClientCACerts {
-		err := cert.Init()
+		err := cert.Init(ctx)
 		if err != nil {
 			return err
 		}

pkg/serverconfigs/sslconfigs/ssl_policy_test.go

@@ -120,7 +120,7 @@ Z3NIV2eNt6YBwkC69DzdazXT
 		OCSPExpiresAt: nowTime + 2,
 	})
 
-	err := policy.Init()
+	err := policy.Init(nil)
 	if err != nil {
 		t.Fatal(err)
 	}