Skip to content

Commit b6eb4a4

Browse files
dessayadessaya
committed
add bloom filter rocksdb config
1 parent 3f79b37 commit b6eb4a4

File tree

19 files changed

+92
-65
lines changed

19 files changed

+92
-65
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ compile-solidity:
2626
cd packages/evm/evmtest/wiki_how_tos && go generate
2727

2828
build-cli:
29-
cd tools/wasp-cli && go mod tidy && go build -ldflags $(BUILD_LD_FLAGS) -tags rocksdb -o ../../
29+
cd tools/wasp-cli && go mod tidy && go build -tags $(BUILD_TAGS) -ldflags $(BUILD_LD_FLAGS) -tags rocksdb -o ../../
3030

3131
build-full: build-cli
3232
$(BUILD_CMD) ./...
@@ -52,7 +52,7 @@ test-cluster: install
5252
go test -tags $(BUILD_TAGS) -race -ldflags $(BUILD_LD_FLAGS) --count 1 -timeout 25m -failfast $(shell go list ./tools/cluster/tests/...)
5353

5454
install-cli:
55-
cd tools/wasp-cli && go mod tidy && go install -ldflags $(BUILD_LD_FLAGS)
55+
cd tools/wasp-cli && go mod tidy && go install -tags $(BUILD_TAGS) -ldflags $(BUILD_LD_FLAGS)
5656

5757
install-full: install-cli
5858
$(INSTALL_CMD) ./...

‎components/database/component.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func provide(c *dig.Container) error {
9090
database.WithEngine(deps.DatabaseEngine),
9191
database.WithPath(path),
9292
database.WithCacheSize(ParamsDatabase.ChainState.CacheSize),
93+
database.WithBloomFilter(ParamsDatabase.ChainState.BloomFilterBitsPerKey),
9394
)
9495
if err != nil {
9596
Component.LogPanic(err.Error())

‎components/database/params.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ type ParametersDatabase struct {
1313
// Path defines the path to the chain state databases folder.
1414
Path string `default:"waspdb/chains/data" usage:"the path to the chain state databases folder"`
1515

16-
CacheSize uint64 `default:"33554432" usage:"size of the RocksDB block cache"`
16+
CacheSize uint64 `default:"33554432" usage:"size of the RocksDB block cache"`
17+
BloomFilterBitsPerKey float64 `default:"0" usage:"RocksDB bloom filter bits per key"`
1718
}
1819

1920
// DebugSkipHealthCheck defines whether to ignore the check for corrupted databases.

‎config_defaults.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
"engine": "rocksdb",
3434
"chainState": {
3535
"path": "waspdb/chains/data",
36-
"cacheSize": 33554432
36+
"cacheSize": 33554432,
37+
"bloomFilterBitsPerKey": 0
3738
},
3839
"debugSkipHealthCheck": true,
3940
"readOnlyFilePath": ""

‎documentation/docs/configuration.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,11 @@ Example:
150150

151151
### <a id="db_chainstate"></a> ChainState
152152

153-
| Name | Description | Type | Default value |
154-
| --------- | -------------------------------------------- | ------ | -------------------- |
155-
| path | The path to the chain state databases folder | string | "waspdb/chains/data" |
156-
| cacheSize | Size of the RocksDB block cache | uint | 33554432 |
153+
| Name | Description | Type | Default value |
154+
| --------------------- | -------------------------------------------- | ------ | -------------------- |
155+
| path | The path to the chain state databases folder | string | "waspdb/chains/data" |
156+
| cacheSize | Size of the RocksDB block cache | uint | 33554432 |
157+
| bloomFilterBitsPerKey | RocksDB bloom filter bits per key | float | 0.0 |
157158

158159
Example:
159160

@@ -163,7 +164,8 @@ Example:
163164
"engine": "rocksdb",
164165
"chainState": {
165166
"path": "waspdb/chains/data",
166-
"cacheSize": 33554432
167+
"cacheSize": 33554432,
168+
"bloomFilterBitsPerKey": 0
167169
},
168170
"debugSkipHealthCheck": true,
169171
"readOnlyFilePath": ""

‎packages/database/database.go‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"github.com/iotaledger/hive.go/runtime/ioutils"
1111
"github.com/iotaledger/wasp/v2/packages/chaindb"
1212
"github.com/iotaledger/wasp/v2/packages/kvstore"
13-
"github.com/iotaledger/wasp/v2/packages/kvstore/rocksdb"
1413
)
1514

1615
var AllowedEngines = []hivedb.Engine{
@@ -89,7 +88,7 @@ func CheckEngine(dbPath string, createDatabaseIfNotExists bool, dbEngine hivedb.
8988
}
9089

9190
func NewDatabaseInMemory() (*Database, error) {
92-
return NewDatabase(hivedb.EngineMapDB, "", false, 0)
91+
return newDatabaseMapDB(), nil
9392
}
9493

9594
// NewDatabase opens a database.
@@ -99,6 +98,7 @@ func NewDatabase(
9998
path string,
10099
createDatabaseIfNotExists bool,
101100
cacheSize uint64,
101+
bloomFilterBitsPerKey float64,
102102
) (*Database, error) {
103103
targetEngine, err := CheckEngine(path, createDatabaseIfNotExists, dbEngine)
104104
if err != nil {
@@ -107,7 +107,11 @@ func NewDatabase(
107107

108108
switch targetEngine {
109109
case hivedb.EngineRocksDB:
110-
return newDatabaseRocksDB(path, cacheSize)
110+
return newDatabaseRocksDB(
111+
path,
112+
cacheSize,
113+
bloomFilterBitsPerKey,
114+
)
111115

112116
case hivedb.EngineMapDB:
113117
return newDatabaseMapDB(), nil
@@ -117,20 +121,6 @@ func NewDatabase(
117121
}
118122
}
119123

120-
func NewReadOnlyDatabase(
121-
path string,
122-
) (*Database, error) {
123-
dbConn, err := rocksdb.OpenDBReadOnly(path)
124-
if err != nil {
125-
return nil, fmt.Errorf("failed to open read-only RocksDB: %w", err)
126-
}
127-
128-
db := New(path, rocksdb.New(dbConn), hivedb.EngineRocksDB, false, func() bool {
129-
return false
130-
})
131-
return db, nil
132-
}
133-
134124
type databaseWithHealthTracker struct {
135125
database *Database
136126
storeHealthTracker *kvstore.StoreHealthTracker
@@ -140,10 +130,11 @@ func newDatabaseWithHealthTracker(
140130
path string,
141131
dbEngine hivedb.Engine,
142132
cacheSize uint64,
133+
bloomFilterBitsPerKey float64,
143134
storeVersion byte,
144135
storeVersionUpdateFunc StoreVersionUpdateFunc,
145136
) (*databaseWithHealthTracker, error) {
146-
db, err := NewDatabase(dbEngine, path, true, cacheSize)
137+
db, err := NewDatabase(dbEngine, path, true, cacheSize, bloomFilterBitsPerKey)
147138
if err != nil {
148139
return nil, err
149140
}

‎packages/database/manager.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ type ChainStateDatabaseManager struct {
2626
mutex sync.RWMutex
2727

2828
// options
29-
engine hivedb.Engine
30-
databasePath string
31-
cacheSize uint64
29+
engine hivedb.Engine
30+
databasePath string
31+
cacheSize uint64
32+
bloomFilterBitsPerKey float64
3233

3334
// databases
3435
databases map[isc.ChainID]*databaseWithHealthTracker
@@ -52,6 +53,12 @@ func WithCacheSize(cacheSize uint64) options.Option[ChainStateDatabaseManager] {
5253
}
5354
}
5455

56+
func WithBloomFilter(bitsPerKey float64) options.Option[ChainStateDatabaseManager] {
57+
return func(d *ChainStateDatabaseManager) {
58+
d.bloomFilterBitsPerKey = bitsPerKey
59+
}
60+
}
61+
5562
func NewChainStateDatabaseManager(chainRecordRegistryProvider registry.ChainRecordRegistryProvider, opts ...options.Option[ChainStateDatabaseManager]) (*ChainStateDatabaseManager, error) {
5663
m := options.Apply(&ChainStateDatabaseManager{
5764
engine: hivedb.EngineAuto,
@@ -103,6 +110,7 @@ func (m *ChainStateDatabaseManager) createDatabase(chainID isc.ChainID) (*databa
103110
path.Join(m.databasePath, chainID.String()),
104111
m.engine,
105112
m.cacheSize,
113+
m.bloomFilterBitsPerKey,
106114
StoreVersionChainState,
107115
nil,
108116
)

‎packages/database/rocksdb.go‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,15 @@ import (
99
)
1010

1111
// NewRocksDB creates a new RocksDB instance.
12-
func NewRocksDB(path string, cacheSize uint64) (*rocksdb.RocksDB, error) {
12+
func NewRocksDB(
13+
path string,
14+
cacheSize uint64,
15+
bloomFilterBitsPerKey float64,
16+
) (*rocksdb.RocksDB, error) {
1317
opts := []rocksdb.Option{
1418
rocksdb.IncreaseParallelism(runtime.NumCPU() - 1),
1519
rocksdb.BlockCacheSize(cacheSize),
20+
rocksdb.BloomFilterBitsPerKey(bloomFilterBitsPerKey),
1621
rocksdb.Custom([]string{
1722
"stats_dump_period_sec=10",
1823
"periodic_compaction_seconds=43200",
@@ -25,8 +30,12 @@ func NewRocksDB(path string, cacheSize uint64) (*rocksdb.RocksDB, error) {
2530
return rocksdb.CreateDB(path, opts...)
2631
}
2732

28-
func newDatabaseRocksDB(path string, cacheSize uint64) (*Database, error) {
29-
rocksDatabase, err := NewRocksDB(path, cacheSize)
33+
func newDatabaseRocksDB(
34+
path string,
35+
cacheSize uint64,
36+
bloomFilterBitsPerKey float64,
37+
) (*Database, error) {
38+
rocksDatabase, err := NewRocksDB(path, cacheSize, bloomFilterBitsPerKey)
3039
if err != nil {
3140
return nil, fmt.Errorf("rocksdb database initialization failed: %w", err)
3241
}
@@ -42,8 +51,21 @@ func newDatabaseRocksDB(path string, cacheSize uint64) (*Database, error) {
4251
running := numCompactions != 0
4352
return running
4453
}
45-
4654
return false
4755
},
4856
), nil
4957
}
58+
59+
func NewReadOnlyDatabase(
60+
path string,
61+
) (*Database, error) {
62+
dbConn, err := rocksdb.OpenDBReadOnly(path)
63+
if err != nil {
64+
return nil, fmt.Errorf("failed to open read-only RocksDB: %w", err)
65+
}
66+
67+
db := New(path, rocksdb.New(dbConn), hivedb.EngineRocksDB, false, func() bool {
68+
return false
69+
})
70+
return db, nil
71+
}

‎packages/evm/jsonrpc/index.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,15 @@ type Index struct {
3131
mu sync.RWMutex
3232
}
3333

34+
// TODO: enable from configuration
35+
const bloomFilterDisabled = 0
36+
3437
func NewIndex(
3538
stateByTrieRoot func(trieRoot trie.Hash) (state.State, error),
3639
indexDBEngine hivedb.Engine,
3740
indexDBPath string,
3841
) *Index {
39-
db, err := database.NewDatabase(indexDBEngine, indexDBPath, true, database.CacheSizeDefault)
42+
db, err := database.NewDatabase(indexDBEngine, indexDBPath, true, database.CacheSizeDefault, bloomFilterDisabled)
4043
if err != nil {
4144
panic(err)
4245
}

‎packages/kvstore/rocksdb/rocksdb.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
//go:build rocksdb
2-
31
package rocksdb
42

53
import (

0 commit comments

Comments
 (0)