TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-04 07:40:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

bfs：实现对FileHeader中Block数据的compaction

Browse Source
This commit is contained in:
刘祥超
2024-04-28 15:38:13 +08:00
parent b272de2122
commit 0dbbc12eb7
2 changed files with 158 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
internal/utils/bfs/file_header.go
View File

@@ -51,19 +51,89 @@ func (this *FileHeader) MaxOffset() int64 {
return 0
}
// Compact blocks
func (this *FileHeader) Compact() {
// TODO 合并相邻的headerBlocks和bodyBlocks（必须是对应的BFile offset也要相邻）
this.compactHeader()
this.compactBody()
}
if len(this.BodyBlocks) > 0 {
sort.Slice(this.BodyBlocks, func(i, j int) bool {
var block1 = this.BodyBlocks[i]
var block2 = this.BodyBlocks[j]
if block1.OriginOffsetFrom == block1.OriginOffsetFrom {
return block1.OriginOffsetTo < block2.OriginOffsetTo
// compact header blocks
func (this *FileHeader) compactHeader() {
var l = len(this.HeaderBlocks)
if l > 1 {
// 合并
var newBlocks []BlockInfo
var newIndex int
for index, currentBlock := range this.HeaderBlocks {
if index == 0 {
newBlocks = append(newBlocks, currentBlock)
newIndex++
continue
}
return block1.OriginOffsetFrom < block2.OriginOffsetFrom
})
var lastBlock = newBlocks[newIndex-1]
if currentBlock.OriginOffsetFrom >= lastBlock.OriginOffsetFrom &&
currentBlock.OriginOffsetFrom <= /* MUST gte */ lastBlock.OriginOffsetTo &&
currentBlock.OriginOffsetFrom-lastBlock.OriginOffsetFrom == currentBlock.BFileOffsetFrom-lastBlock.BFileOffsetFrom /* 两侧距离一致 */ {
if currentBlock.OriginOffsetTo > lastBlock.OriginOffsetTo {
lastBlock.OriginOffsetTo = currentBlock.OriginOffsetTo
lastBlock.BFileOffsetTo = currentBlock.BFileOffsetTo
newBlocks[newIndex-1] = lastBlock
}
} else {
newBlocks = append(newBlocks, currentBlock)
newIndex++
}
}
this.HeaderBlocks = newBlocks
}
}
// sort and compact body blocks
func (this *FileHeader) compactBody() {
var l = len(this.BodyBlocks)
if l > 0 {
if l > 1 {
// 排序
sort.Slice(this.BodyBlocks, func(i, j int) bool {
var block1 = this.BodyBlocks[i]
var block2 = this.BodyBlocks[j]
if block1.OriginOffsetFrom == block1.OriginOffsetFrom {
return block1.OriginOffsetTo < block2.OriginOffsetTo
}
return block1.OriginOffsetFrom < block2.OriginOffsetFrom
})
// 合并
var newBlocks []BlockInfo
var newIndex int
for index, currentBlock := range this.BodyBlocks {
if index == 0 {
newBlocks = append(newBlocks, currentBlock)
newIndex++
continue
}
var lastBlock = newBlocks[newIndex-1]
if currentBlock.OriginOffsetFrom >= lastBlock.OriginOffsetFrom &&
currentBlock.OriginOffsetFrom <= /* MUST gte */ lastBlock.OriginOffsetTo &&
currentBlock.OriginOffsetFrom-lastBlock.OriginOffsetFrom == currentBlock.BFileOffsetFrom-lastBlock.BFileOffsetFrom /* 两侧距离一致 */ {
if currentBlock.OriginOffsetTo > lastBlock.OriginOffsetTo {
lastBlock.OriginOffsetTo = currentBlock.OriginOffsetTo
lastBlock.BFileOffsetTo = currentBlock.BFileOffsetTo
newBlocks[newIndex-1] = lastBlock
}
} else {
newBlocks = append(newBlocks, currentBlock)
newIndex++
}
}
this.BodyBlocks = newBlocks
l = len(this.BodyBlocks)
}
// 检查是否已完成
var isCompleted = true
if this.BodyBlocks[0].OriginOffsetFrom != 0 || this.BodyBlocks[len(this.BodyBlocks)-1].OriginOffsetTo != this.BodySize {
isCompleted = false
@@ -80,6 +150,7 @@ func (this *FileHeader) Compact() {
}
}
// Clone current header
func (this *FileHeader) Clone() *FileHeader {
return &FileHeader{
Version: this.Version,

78
internal/utils/bfs/file_header_test.go
View File

@@ -141,6 +141,84 @@ func TestFileHeader_Compact(t *testing.T) {
}
}
func TestFileHeader_Compact_Merge(t *testing.T) {
var a = assert.NewAssertion(t)
var header = &bfs.FileHeader{
Version: 1,
Status: 200,
HeaderBlocks: []bfs.BlockInfo{
{
BFileOffsetFrom: 1000,
BFileOffsetTo: 1100,
OriginOffsetFrom: 1200,
OriginOffsetTo: 1300,
},
{
BFileOffsetFrom: 1100,
BFileOffsetTo: 1200,
OriginOffsetFrom: 1300,
OriginOffsetTo: 1400,
},
},
BodyBlocks: []bfs.BlockInfo{
{
BFileOffsetFrom: 0,
BFileOffsetTo: 100,
OriginOffsetFrom: 200,
OriginOffsetTo: 300,
},
{
BFileOffsetFrom: 100,
BFileOffsetTo: 200,
OriginOffsetFrom: 300,
OriginOffsetTo: 400,
},
{
BFileOffsetFrom: 200,
BFileOffsetTo: 300,
OriginOffsetFrom: 400,
OriginOffsetTo: 500,
},
},
}
header.Compact()
logs.PrintAsJSON(header.HeaderBlocks)
logs.PrintAsJSON(header.BodyBlocks)
a.IsTrue(len(header.HeaderBlocks) == 1)
a.IsTrue(len(header.BodyBlocks) == 1)
}
func TestFileHeader_Compact_Merge2(t *testing.T) {
var header = &bfs.FileHeader{
Version: 1,
Status: 200,
BodyBlocks: []bfs.BlockInfo{
{
BFileOffsetFrom: 0,
BFileOffsetTo: 100,
OriginOffsetFrom: 200,
OriginOffsetTo: 300,
},
{
BFileOffsetFrom: 101,
BFileOffsetTo: 200,
OriginOffsetFrom: 301,
OriginOffsetTo: 400,
},
{
BFileOffsetFrom: 200,
BFileOffsetTo: 300,
OriginOffsetFrom: 400,
OriginOffsetTo: 500,
},
},
}
header.Compact()
logs.PrintAsJSON(header.BodyBlocks)
}
func TestFileHeader_Clone(t *testing.T) {
var a = assert.NewAssertion(t)