数据有更改时发送通知

2025-12-30 09:56:35 +08:00 · 2021-08-08 15:47:48 +08:00
parent 07aa0990f8
commit e979414973
40 changed files with 548 additions and 185 deletions
--- a/internal/rpc/services/nameservers/service_ns.go
+++ b/internal/rpc/services/nameservers/service_ns.go
@@ -45,21 +45,21 @@ func (this *NSService) ComposeNSBoard(ctx context.Context, req *pb.ComposeNSBoar
 	result.CountNSRecords = countRecords

 	// 集群数
-	countClusters, err := nameservers.SharedNSClusterDAO.CountAllEnabledClusters(tx)
+	countClusters, err := models.SharedNSClusterDAO.CountAllEnabledClusters(tx)
 	if err != nil {
 		return nil, err
 	}
 	result.CountNSClusters = countClusters

 	// 节点数
-	countNodes, err := nameservers.SharedNSNodeDAO.CountAllEnabledNodes(tx)
+	countNodes, err := models.SharedNSNodeDAO.CountAllEnabledNodes(tx)
 	if err != nil {
 		return nil, err
 	}
 	result.CountNSNodes = countNodes

 	// 离线节点数
-	countOfflineNodes, err := nameservers.SharedNSNodeDAO.CountAllOfflineNodes(tx)
+	countOfflineNodes, err := models.SharedNSNodeDAO.CountAllOfflineNodes(tx)
 	if err != nil {
 		return nil, err
 	}
@@ -122,7 +122,7 @@ func (this *NSService) ComposeNSBoard(ctx context.Context, req *pb.ComposeNSBoar
 		return nil, err
 	}
 	for _, stat := range topNodeStats {
-		nodeName, err := nameservers.SharedNSNodeDAO.FindEnabledNSNodeName(tx, int64(stat.NodeId))
+		nodeName, err := models.SharedNSNodeDAO.FindEnabledNSNodeName(tx, int64(stat.NodeId))
 		if err != nil {
 			return nil, err
 		}
--- a/internal/rpc/services/nameservers/service_ns_cluster.go
+++ b/internal/rpc/services/nameservers/service_ns_cluster.go
@@ -4,8 +4,9 @@ package nameservers

 import (
 	"context"
-	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 )

@@ -21,7 +22,7 @@ func (this *NSClusterService) CreateNSCluster(ctx context.Context, req *pb.Creat
 		return nil, err
 	}
 	var tx = this.NullTx()
-	clusterId, err := nameservers.SharedNSClusterDAO.CreateCluster(tx, req.Name, req.AccessLogJSON)
+	clusterId, err := models.SharedNSClusterDAO.CreateCluster(tx, req.Name, req.AccessLogJSON)
 	if err != nil {
 		return nil, err
 	}
@@ -35,7 +36,7 @@ func (this *NSClusterService) UpdateNSCluster(ctx context.Context, req *pb.Updat
 		return nil, err
 	}
 	var tx = this.NullTx()
-	err = nameservers.SharedNSClusterDAO.UpdateCluster(tx, req.NsClusterId, req.Name, req.IsOn)
+	err = models.SharedNSClusterDAO.UpdateCluster(tx, req.NsClusterId, req.Name, req.IsOn)
 	if err != nil {
 		return nil, err
 	}
@@ -50,7 +51,7 @@ func (this *NSClusterService) FindNSClusterAccessLog(ctx context.Context, req *p
 	}

 	var tx = this.NullTx()
-	accessLogJSON, err := nameservers.SharedNSClusterDAO.FindClusterAccessLog(tx, req.NsClusterId)
+	accessLogJSON, err := models.SharedNSClusterDAO.FindClusterAccessLog(tx, req.NsClusterId)
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +66,7 @@ func (this *NSClusterService) UpdateNSClusterAccessLog(ctx context.Context, req
 	}

 	var tx = this.NullTx()
-	err = nameservers.SharedNSClusterDAO.UpdateClusterAccessLog(tx, req.NsClusterId, req.AccessLogJSON)
+	err = models.SharedNSClusterDAO.UpdateClusterAccessLog(tx, req.NsClusterId, req.AccessLogJSON)
 	if err != nil {
 		return nil, err
 	}
@@ -79,10 +80,17 @@ func (this *NSClusterService) DeleteNSCluster(ctx context.Context, req *pb.Delet
 		return nil, err
 	}
 	var tx = this.NullTx()
-	err = nameservers.SharedNSClusterDAO.DisableNSCluster(tx, req.NsClusterId)
+	err = models.SharedNSClusterDAO.DisableNSCluster(tx, req.NsClusterId)
 	if err != nil {
 		return nil, err
 	}
+
+	// 删除任务
+	err = models.SharedNodeTaskDAO.DeleteAllClusterTasks(tx, nodeconfigs.NodeRoleDNS, req.NsClusterId)
+	if err != nil {
+		return nil, err
+	}
+
 	return this.Success()
 }

@@ -93,7 +101,7 @@ func (this *NSClusterService) FindEnabledNSCluster(ctx context.Context, req *pb.
 		return nil, err
 	}
 	var tx = this.NullTx()
-	cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, req.NsClusterId)
+	cluster, err := models.SharedNSClusterDAO.FindEnabledNSCluster(tx, req.NsClusterId)
 	if err != nil {
 		return nil, err
 	}
@@ -115,7 +123,7 @@ func (this *NSClusterService) CountAllEnabledNSClusters(ctx context.Context, req
 		return nil, err
 	}
 	var tx = this.NullTx()
-	count, err := nameservers.SharedNSClusterDAO.CountAllEnabledClusters(tx)
+	count, err := models.SharedNSClusterDAO.CountAllEnabledClusters(tx)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +137,7 @@ func (this *NSClusterService) ListEnabledNSClusters(ctx context.Context, req *pb
 		return nil, err
 	}
 	var tx = this.NullTx()
-	clusters, err := nameservers.SharedNSClusterDAO.ListEnabledClusters(tx, req.Offset, req.Size)
+	clusters, err := models.SharedNSClusterDAO.ListEnabledClusters(tx, req.Offset, req.Size)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +160,7 @@ func (this *NSClusterService) FindAllEnabledNSClusters(ctx context.Context, req
 		return nil, err
 	}
 	var tx = this.NullTx()
-	clusters, err := nameservers.SharedNSClusterDAO.FindAllEnabledClusters(tx)
+	clusters, err := models.SharedNSClusterDAO.FindAllEnabledClusters(tx)
 	if err != nil {
 		return nil, err
 	}
--- a/internal/rpc/services/nameservers/service_ns_domain.go
+++ b/internal/rpc/services/nameservers/service_ns_domain.go
@@ -78,7 +78,7 @@ func (this *NSDomainService) FindEnabledNSDomain(ctx context.Context, req *pb.Fi
 	}

 	// 集群
-	cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(domain.ClusterId))
+	cluster, err := models.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(domain.ClusterId))
 	if err != nil {
 		return nil, err
 	}
@@ -150,7 +150,7 @@ func (this *NSDomainService) ListEnabledNSDomains(ctx context.Context, req *pb.L
 	pbDomains := []*pb.NSDomain{}
 	for _, domain := range domains {
 		// 集群
-		cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(domain.ClusterId))
+		cluster, err := models.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(domain.ClusterId))
 		if err != nil {
 			return nil, err
 		}
--- a/internal/rpc/services/nameservers/service_ns_node.go
+++ b/internal/rpc/services/nameservers/service_ns_node.go
@@ -5,12 +5,13 @@ package nameservers
 import (
 	"context"
 	"encoding/json"
-	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/installers"
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	stringutil "github.com/iwind/TeaGo/utils/string"
 	"path/filepath"
@@ -30,7 +31,7 @@ func (this *NSNodeService) FindAllEnabledNSNodesWithNSClusterId(ctx context.Cont

 	var tx = this.NullTx()

-	nodes, err := nameservers.SharedNSNodeDAO.FindAllEnabledNodesWithClusterId(tx, req.NsClusterId)
+	nodes, err := models.SharedNSNodeDAO.FindAllEnabledNodesWithClusterId(tx, req.NsClusterId)
 	if err != nil {
 		return nil, err
 	}
@@ -60,7 +61,7 @@ func (this *NSNodeService) CountAllEnabledNSNodes(ctx context.Context, req *pb.C
 	}

 	var tx = this.NullTx()
-	count, err := nameservers.SharedNSNodeDAO.CountAllEnabledNodes(tx)
+	count, err := models.SharedNSNodeDAO.CountAllEnabledNodes(tx)
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +76,7 @@ func (this *NSNodeService) CountAllEnabledNSNodesMatch(ctx context.Context, req
 	}

 	var tx = this.NullTx()
-	count, err := nameservers.SharedNSNodeDAO.CountAllEnabledNodesMatch(tx, req.NsClusterId, configutils.ToBoolState(req.InstallState), configutils.ToBoolState(req.ActiveState), req.Keyword)
+	count, err := models.SharedNSNodeDAO.CountAllEnabledNodesMatch(tx, req.NsClusterId, configutils.ToBoolState(req.InstallState), configutils.ToBoolState(req.ActiveState), req.Keyword)
 	if err != nil {
 		return nil, err
 	}
@@ -90,7 +91,7 @@ func (this *NSNodeService) ListEnabledNSNodesMatch(ctx context.Context, req *pb.
 	}

 	var tx = this.NullTx()
-	nodes, err := nameservers.SharedNSNodeDAO.ListAllEnabledNodesMatch(tx, req.NsClusterId, configutils.ToBoolState(req.InstallState), configutils.ToBoolState(req.ActiveState), req.Keyword, req.Offset, req.Size)
+	nodes, err := models.SharedNSNodeDAO.ListAllEnabledNodesMatch(tx, req.NsClusterId, configutils.ToBoolState(req.InstallState), configutils.ToBoolState(req.ActiveState), req.Keyword, req.Offset, req.Size)
 	if err != nil {
 		return nil, err
 	}
@@ -144,7 +145,7 @@ func (this *NSNodeService) CountAllUpgradeNSNodesWithNSClusterId(ctx context.Con
 	deployFiles := installers.SharedDeployManager.LoadNSNodeFiles()
 	total := int64(0)
 	for _, deployFile := range deployFiles {
-		count, err := nameservers.SharedNSNodeDAO.CountAllLowerVersionNodesWithClusterId(tx, req.NsClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
+		count, err := models.SharedNSNodeDAO.CountAllLowerVersionNodesWithClusterId(tx, req.NsClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
 		if err != nil {
 			return nil, err
 		}
@@ -163,7 +164,7 @@ func (this *NSNodeService) CreateNSNode(ctx context.Context, req *pb.CreateNSNod

 	tx := this.NullTx()

-	nodeId, err := nameservers.SharedNSNodeDAO.CreateNode(tx, adminId, req.Name, req.NodeClusterId)
+	nodeId, err := models.SharedNSNodeDAO.CreateNode(tx, adminId, req.Name, req.NodeClusterId)
 	if err != nil {
 		return nil, err
 	}
@@ -182,7 +183,13 @@ func (this *NSNodeService) DeleteNSNode(ctx context.Context, req *pb.DeleteNSNod

 	tx := this.NullTx()

-	err = nameservers.SharedNSNodeDAO.DisableNSNode(tx, req.NsNodeId)
+	err = models.SharedNSNodeDAO.DisableNSNode(tx, req.NsNodeId)
+	if err != nil {
+		return nil, err
+	}
+
+	// 删除任务
+	err = models.SharedNodeTaskDAO.DeleteNodeTasks(tx, nodeconfigs.NodeRoleDNS, req.NsNodeId)
 	if err != nil {
 		return nil, err
 	}
@@ -199,7 +206,7 @@ func (this *NSNodeService) FindEnabledNSNode(ctx context.Context, req *pb.FindEn

 	tx := this.NullTx()

-	node, err := nameservers.SharedNSNodeDAO.FindEnabledNSNode(tx, req.NsNodeId)
+	node, err := models.SharedNSNodeDAO.FindEnabledNSNode(tx, req.NsNodeId)
 	if err != nil {
 		return nil, err
 	}
@@ -208,7 +215,7 @@ func (this *NSNodeService) FindEnabledNSNode(ctx context.Context, req *pb.FindEn
 	}

 	// 集群信息
-	clusterName, err := nameservers.SharedNSClusterDAO.FindEnabledNSClusterName(tx, int64(node.ClusterId))
+	clusterName, err := models.SharedNSClusterDAO.FindEnabledNSClusterName(tx, int64(node.ClusterId))
 	if err != nil {
 		return nil, err
 	}
@@ -256,7 +263,7 @@ func (this *NSNodeService) UpdateNSNode(ctx context.Context, req *pb.UpdateNSNod

 	tx := this.NullTx()

-	err = nameservers.SharedNSNodeDAO.UpdateNode(tx, req.NsNodeId, req.Name, req.NsClusterId, req.IsOn)
+	err = models.SharedNSNodeDAO.UpdateNode(tx, req.NsNodeId, req.Name, req.NsClusterId, req.IsOn)
 	if err != nil {
 		return nil, err
 	}
@@ -287,7 +294,7 @@ func (this *NSNodeService) FindNSNodeInstallStatus(ctx context.Context, req *pb.

 	tx := this.NullTx()

-	installStatus, err := nameservers.SharedNSNodeDAO.FindNodeInstallStatus(tx, req.NsNodeId)
+	installStatus, err := models.SharedNSNodeDAO.FindNodeInstallStatus(tx, req.NsNodeId)
 	if err != nil {
 		return nil, err
 	}
@@ -315,7 +322,7 @@ func (this *NSNodeService) UpdateNSNodeIsInstalled(ctx context.Context, req *pb.

 	tx := this.NullTx()

-	err = nameservers.SharedNSNodeDAO.UpdateNodeIsInstalled(tx, req.NsNodeId, req.IsInstalled)
+	err = models.SharedNSNodeDAO.UpdateNodeIsInstalled(tx, req.NsNodeId, req.IsInstalled)
 	if err != nil {
 		return nil, err
 	}
@@ -341,7 +348,7 @@ func (this *NSNodeService) UpdateNSNodeStatus(ctx context.Context, req *pb.Updat

 	tx := this.NullTx()

-	err = nameservers.SharedNSNodeDAO.UpdateNodeStatus(tx, nodeId, req.StatusJSON)
+	err = models.SharedNSNodeDAO.UpdateNodeStatus(tx, nodeId, req.StatusJSON)
 	if err != nil {
 		return nil, err
 	}
@@ -357,7 +364,7 @@ func (this *NSNodeService) FindCurrentNSNodeConfig(ctx context.Context, req *pb.
 	}

 	var tx = this.NullTx()
-	config, err := nameservers.SharedNSNodeDAO.ComposeNodeConfig(tx, nodeId)
+	config, err := models.SharedNSNodeDAO.ComposeNodeConfig(tx, nodeId)
 	if err != nil {
 		return nil, err
 	}
@@ -432,7 +439,7 @@ func (this *NSNodeService) UpdateNSNodeConnectedAPINodes(ctx context.Context, re

 	tx := this.NullTx()

-	err = nameservers.SharedNSNodeDAO.UpdateNodeConnectedAPINodes(tx, nodeId, req.ApiNodeIds)
+	err = models.SharedNSNodeDAO.UpdateNodeConnectedAPINodes(tx, nodeId, req.ApiNodeIds)
 	if err != nil {
 		return nil, errors.Wrap(err)
 	}
--- a/internal/rpc/services/nameservers/service_ns_node_stream.go
+++ b/internal/rpc/services/nameservers/service_ns_node_stream.go
@@ -6,12 +6,13 @@ import (
 	"fmt"
 	"github.com/TeaOSLab/EdgeAPI/internal/configs"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
-	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/logs"
 	"strconv"
 	"sync"
@@ -48,22 +49,56 @@ var requestChanMap = map[int64]chan *CommandRequest{} // node id => chan
 func NextCommandRequestId() int64 {
 	return atomic.AddInt64(&commandRequestId, 1)
 }
-
 func init() {
-	// 清理WaitingChannelMap
-	ticker := time.NewTicker(30 * time.Second)
-	go func() {
-		for range ticker.C {
-			nodeLocker.Lock()
-			for requestId, request := range responseChanMap {
-				if time.Now().Unix()-request.Timestamp > 3600 {
-					responseChanMap[requestId].Close()
-					delete(responseChanMap, requestId)
+	dbs.OnReadyDone(func() {
+		// 清理WaitingChannelMap
+		go func() {
+			ticker := time.NewTicker(30 * time.Second)
+			for range ticker.C {
+				nodeLocker.Lock()
+				for requestId, request := range responseChanMap {
+					if time.Now().Unix()-request.Timestamp > 3600 {
+						responseChanMap[requestId].Close()
+						delete(responseChanMap, requestId)
+					}
 				}
+				nodeLocker.Unlock()
 			}
-			nodeLocker.Unlock()
-		}
-	}()
+		}()
+
+		// 自动同步连接到本API节点的NS节点任务
+		go func() {
+			defer func() {
+				_ = recover()
+			}()
+
+			// TODO 未来支持同步边缘节点
+			var ticker = time.NewTicker(3 * time.Second)
+			for range ticker.C {
+				nodeIds, err := models.SharedNodeTaskDAO.FindAllDoingNodeIds(nil, nodeconfigs.NodeRoleDNS)
+				if err != nil {
+					remotelogs.Error("NSNodeService_SYNC", err.Error())
+					continue
+				}
+				nodeLocker.Lock()
+				for _, nodeId := range nodeIds {
+					c, ok := requestChanMap[nodeId]
+					if ok {
+						select {
+						case c <- &CommandRequest{
+							Id:          NextCommandRequestId(),
+							Code:        messageconfigs.NSMessageCodeNewNodeTask,
+							CommandJSON: nil,
+						}:
+						default:
+
+						}
+					}
+				}
+				nodeLocker.Unlock()
+			}
+		}()
+	})
 }

 // NsNodeStream 节点stream
@@ -100,22 +135,22 @@ func (this *NSNodeService) NsNodeStream(server pb.NSNodeService_NsNodeStreamServ
 	tx := this.NullTx()

 	// 标记为活跃状态
-	oldIsActive, err := nameservers.SharedNSNodeDAO.FindNodeActive(tx, nodeId)
+	oldIsActive, err := models.SharedNSNodeDAO.FindNodeActive(tx, nodeId)
 	if err != nil {
 		return err
 	}
 	if !oldIsActive {
-		err = nameservers.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, true)
+		err = models.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, true)
 		if err != nil {
 			return err
 		}

 		// 发送恢复消息
-		clusterId, err := nameservers.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
+		clusterId, err := models.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
 		if err != nil {
 			return err
 		}
-		nodeName, err := nameservers.SharedNSNodeDAO.FindEnabledNSNodeName(tx, nodeId)
+		nodeName, err := models.SharedNSNodeDAO.FindEnabledNSNodeName(tx, nodeId)
 		if err != nil {
 			return err
 		}
@@ -169,7 +204,7 @@ func (this *NSNodeService) NsNodeStream(server pb.NSNodeService_NsNodeStreamServ
 		req, err := server.Recv()
 		if err != nil {
 			// 修改节点状态
-			err1 := nameservers.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, false)
+			err1 := models.SharedNSNodeDAO.UpdateNodeActive(tx, nodeId, false)
 			if err1 != nil {
 				logs.Println(err1.Error())
 			}
--- a/internal/rpc/services/nameservers/service_ns_record_hourly_stat.go
+++ b/internal/rpc/services/nameservers/service_ns_record_hourly_stat.go
@@ -4,6 +4,7 @@ package nameservers

 import (
 	"context"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
@@ -31,7 +32,7 @@ func (this *NSRecordHourlyStatService) UploadNSRecordHourlyStats(ctx context.Con
 	}

 	var tx = this.NullTx()
-	clusterId, err := nameservers.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
+	clusterId, err := models.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
 	if err != nil {
 		return nil, err
 	}
--- a/internal/rpc/services/nameservers/service_ns_route.go
+++ b/internal/rpc/services/nameservers/service_ns_route.go
@@ -4,6 +4,7 @@ package nameservers

 import (
 	"context"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
@@ -75,7 +76,7 @@ func (this *NSRouteService) FindEnabledNSRoute(ctx context.Context, req *pb.Find
 	// 集群
 	var pbCluster *pb.NSCluster
 	if route.ClusterId > 0 {
-		cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(route.ClusterId))
+		cluster, err := models.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(route.ClusterId))
 		if err != nil {
 			return nil, err
 		}
@@ -130,7 +131,7 @@ func (this *NSRouteService) FindAllEnabledNSRoutes(ctx context.Context, req *pb.
 		// 集群
 		var pbCluster *pb.NSCluster
 		if route.ClusterId > 0 {
-			cluster, err := nameservers.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(route.ClusterId))
+			cluster, err := models.SharedNSClusterDAO.FindEnabledNSCluster(tx, int64(route.ClusterId))
 			if err != nil {
 				return nil, err
 			}