Skip to content

Commit f3827f1

Browse files
cheb0cheb0
committed
remove IsLastLID from storage format
1 parent 757588c commit f3827f1

3 files changed

Lines changed: 55 additions & 109 deletions

File tree

‎frac/sealed/lids/block.go‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,8 @@ func (b *Block) getLIDs(i int) []uint32 {
2424
}
2525

2626
func (b *Block) Pack(dst []byte, buf []uint32) []byte {
27-
if b.IsLastLID {
28-
dst = append(dst, 1)
29-
} else {
30-
dst = append(dst, 0)
31-
}
32-
3327
dst = packer.CompressDeltaBitpackUint32(dst, b.Offsets, buf)
3428
dst = packer.CompressDeltaBitpackUint32(dst, b.LIDs, buf)
35-
3629
return dst
3730
}
3831

@@ -55,13 +48,6 @@ func (b *Block) Unpack(data []byte, fracVer config.BinaryDataVersion, buf *Unpac
5548
}
5649

5750
func (b *Block) unpackBitpack(data []byte, buf *UnpackBuffer) error {
58-
if data[0] == 1 {
59-
b.IsLastLID = true
60-
} else {
61-
b.IsLastLID = false
62-
}
63-
data = data[1:]
64-
6551
var err error
6652
var values []uint32
6753

@@ -71,7 +57,7 @@ func (b *Block) unpackBitpack(data []byte, buf *UnpackBuffer) error {
7157
}
7258
b.Offsets = append([]uint32{}, values...)
7359

74-
data, values, err = packer.DecompressDeltaBitpackUint32(data, buf.decompressed, buf.compressed)
60+
_, values, err = packer.DecompressDeltaBitpackUint32(data, buf.decompressed, buf.compressed)
7561
if err != nil {
7662
return err
7763
}

‎frac/sealed/lids/block_test.go‎

Lines changed: 52 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -16,44 +16,32 @@ func TestBlockPack(t *testing.T) {
1616
name string
1717
lids []uint32
1818
offsets []uint32
19-
isLastLID bool
2019
generator func() ([]uint32, []uint32)
2120
}{
2221
{
23-
name: "small_single_token",
24-
lids: generate(4),
25-
offsets: []uint32{0, 4},
26-
isLastLID: true,
22+
name: "small_single_token",
23+
lids: generate(4),
24+
offsets: []uint32{0, 4},
2725
},
2826
{
29-
name: "small_a_few_token",
30-
lids: generate(6),
31-
offsets: []uint32{0, 3, 6},
32-
isLastLID: true,
27+
name: "small_a_few_token",
28+
lids: generate(6),
29+
offsets: []uint32{0, 3, 6},
3330
},
3431
{
35-
name: "small_not_last_lid",
36-
lids: generate(3),
37-
offsets: []uint32{0, 3},
38-
isLastLID: false,
32+
name: "small_single_lid",
33+
lids: []uint32{100},
34+
offsets: []uint32{0, 1},
3935
},
4036
{
41-
name: "small_single_lid",
42-
lids: []uint32{100},
43-
offsets: []uint32{0, 1},
44-
isLastLID: true,
37+
name: "small_big_lids",
38+
lids: []uint32{math.MaxUint32 - 100, math.MaxUint32 - 50, math.MaxUint32 - 10},
39+
offsets: []uint32{0, 3},
4540
},
4641
{
47-
name: "small_big_lids",
48-
lids: []uint32{math.MaxUint32 - 100, math.MaxUint32 - 50, math.MaxUint32 - 10},
49-
offsets: []uint32{0, 3},
50-
isLastLID: true,
51-
},
52-
{
53-
name: "small_few_tokens",
54-
lids: generate(8),
55-
offsets: []uint32{0, 3, 6, 8},
56-
isLastLID: false,
42+
name: "small_few_tokens",
43+
lids: generate(8),
44+
offsets: []uint32{0, 3, 6, 8},
5745
},
5846
{
5947
name: "medium_many_tokens",
@@ -70,19 +58,6 @@ func TestBlockPack(t *testing.T) {
7058
}
7159
return lids, offsets
7260
},
73-
isLastLID: true,
74-
},
75-
{
76-
name: "large_is_last_lid_false",
77-
generator: func() ([]uint32, []uint32) {
78-
lids := make([]uint32, 0, 200)
79-
startLID := uint32(1000)
80-
for i := 0; i < 200; i++ {
81-
lids = append(lids, startLID+uint32(i*10))
82-
}
83-
return lids, []uint32{0, uint32(len(lids))}
84-
},
85-
isLastLID: false,
8661
},
8762
{
8863
name: "large_many_tokens",
@@ -98,61 +73,51 @@ func TestBlockPack(t *testing.T) {
9873
}
9974
return lids, offsets
10075
},
101-
isLastLID: true,
10276
},
10377
{
104-
name: "medium_128_lids",
105-
lids: generate(128),
106-
offsets: []uint32{0, 128},
107-
isLastLID: true,
78+
name: "medium_128_lids",
79+
lids: generate(128),
80+
offsets: []uint32{0, 128},
10881
},
10982
{
110-
name: "medium_127_lids",
111-
lids: generate(127),
112-
offsets: []uint32{0, 127},
113-
isLastLID: true,
83+
name: "medium_127_lids",
84+
lids: generate(127),
85+
offsets: []uint32{0, 127},
11486
},
11587
{
116-
name: "medium_129_lids",
117-
lids: generate(129),
118-
offsets: []uint32{0, 129},
119-
isLastLID: true,
88+
name: "medium_129_lids",
89+
lids: generate(129),
90+
offsets: []uint32{0, 129},
12091
},
12192
{
122-
name: "medium_4k_lids",
123-
lids: generate(4096),
124-
offsets: []uint32{0, 4096},
125-
isLastLID: true,
93+
name: "medium_4k_lids",
94+
lids: generate(4096),
95+
offsets: []uint32{0, 4096},
12696
},
12797
{
128-
name: "medium_4k_minus_one_lids",
129-
lids: generate(4095),
130-
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 4095},
131-
isLastLID: true,
98+
name: "medium_4k_minus_one_lids",
99+
lids: generate(4095),
100+
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 4095},
132101
},
133102
{
134-
name: "medium_4k_plus_one_lids",
135-
lids: generate(4097),
136-
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 4097},
137-
isLastLID: true,
103+
name: "medium_4k_plus_one_lids",
104+
lids: generate(4097),
105+
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 4097},
138106
},
139107
{
140-
name: "medium_64k_lids",
141-
lids: generate(65536),
142-
offsets: []uint32{0, 65536},
143-
isLastLID: false,
108+
name: "medium_64k_lids",
109+
lids: generate(65536),
110+
offsets: []uint32{0, 65536},
144111
},
145112
{
146-
name: "medium_64k_minus_one_lids",
147-
lids: generate(65535),
148-
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 65535},
149-
isLastLID: true,
113+
name: "medium_64k_minus_one_lids",
114+
lids: generate(65535),
115+
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 65535},
150116
},
151117
{
152-
name: "medium_64k_plus_one_lids",
153-
lids: generate(65537),
154-
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 65537},
155-
isLastLID: false,
118+
name: "medium_64k_plus_one_lids",
119+
lids: generate(65537),
120+
offsets: []uint32{0, 10, 50, 100, 150, 190, 1000, 1500, 65537},
156121
},
157122
}
158123

@@ -169,9 +134,8 @@ func TestBlockPack(t *testing.T) {
169134
}
170135

171136
block := &Block{
172-
LIDs: lids,
173-
Offsets: offsets,
174-
IsLastLID: tc.isLastLID,
137+
LIDs: lids,
138+
Offsets: offsets,
175139
}
176140

177141
packed := block.Pack(nil, nil)
@@ -200,15 +164,13 @@ func generate(n int) []uint32 {
200164
func TestBlockPack_ReuseBuffer(t *testing.T) {
201165
// Test that UnpackBuffer can be reused
202166
block1 := &Block{
203-
LIDs: generate(64 * 1024),
204-
Offsets: []uint32{0, 3},
205-
IsLastLID: true,
167+
LIDs: generate(64 * 1024),
168+
Offsets: []uint32{0, 3},
206169
}
207170

208171
block2 := &Block{
209-
LIDs: generate(64 * 1024),
210-
Offsets: []uint32{0, 4},
211-
IsLastLID: true,
172+
LIDs: generate(64 * 1024),
173+
Offsets: []uint32{0, 4},
212174
}
213175

214176
buf1 := make([]uint32, 0, 64*1024)
@@ -234,9 +196,8 @@ func BenchmarkBlock_Pack(b *testing.B) {
234196
lids := generate(64 * 1024)
235197

236198
block := &Block{
237-
LIDs: lids,
238-
Offsets: []uint32{0, 64 * 1024},
239-
IsLastLID: true,
199+
LIDs: lids,
200+
Offsets: []uint32{0, 64 * 1024},
240201
}
241202
tmp := make([]uint32, 0, 64*1024/4)
242203

@@ -249,9 +210,8 @@ func BenchmarkBlock_Unpack(b *testing.B) {
249210
lids := generate(64 * 1024)
250211

251212
block := &Block{
252-
LIDs: lids,
253-
Offsets: []uint32{0, 64 * 1024},
254-
IsLastLID: true,
213+
LIDs: lids,
214+
Offsets: []uint32{0, 64 * 1024},
255215
}
256216
packed := block.Pack(nil, nil)
257217

‎packer/delta_bitpacker.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func CompressDeltaBitpackUint32(dst []byte, values, buf []uint32) []byte {
3636
return dst
3737
}
3838

39-
func DecompressDeltaBitpackUint32(data []byte, buf []uint32, compressed []uint32) ([]byte, []uint32, error) {
39+
func DecompressDeltaBitpackUint32(data []byte, buf, compressed []uint32) ([]byte, []uint32, error) {
4040
if len(data) < sizeOfUint32 {
4141
return nil, nil, fmt.Errorf("not enough data. slice len %d", len(data))
4242
}
@@ -104,7 +104,7 @@ func CompressDeltaBitpackUint64(dst []byte, values, buf []uint64) []byte {
104104
return dst
105105
}
106106

107-
func DecompressDeltaBitpackUint64(data []byte, buf []uint64, compressed []uint64) ([]byte, []uint64, error) {
107+
func DecompressDeltaBitpackUint64(data []byte, buf, compressed []uint64) ([]byte, []uint64, error) {
108108
if len(data) < 4 {
109109
return nil, nil, fmt.Errorf("not enough data. slice len %d", len(data))
110110
}

0 commit comments

Comments
 (0)