Skip to content

Commit 5ed6385

Browse files
committed
add bloom filter rocksdb config
1 parent 3f79b37 commit 5ed6385

File tree

25 files changed

+103
-131
lines changed

25 files changed

+103
-131
lines changed

CONTRIBUTING.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ To run tests locally, Docker is required to be installed and started.
1717
Execute one of the following commands:
1818

1919
```shell
20-
go test -short -tags rocksdb ./...
20+
go test -short ./...
2121
```
2222

2323
Or, as an alternative:
@@ -108,4 +108,3 @@ Adjust your VS Code settings as follows:
108108
3. Add a `golangci` file watcher with a custom command. We recommend using it with the `--fix` parameter.
109109

110110
![A screenshot that shows how to add a golangci file watcher in GoLand.](/documentation/contributing/golintci-goland-3.png "Click to see the full-sized image.")
111-

Dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ ARG GOLANG_IMAGE_TAG=1.24-bullseye
33

44
# Build stage
55
FROM golang:${GOLANG_IMAGE_TAG} AS build
6-
ARG BUILD_TAGS=rocksdb
76
ARG BUILD_LD_FLAGS="--X=github.com/iotaledger/wasp/v2/components/app.Version=v0.0.0-testing"
87

98
LABEL org.label-schema.description="Wasp"
@@ -34,7 +33,7 @@ COPY . .
3433

3534
RUN --mount=type=cache,target=/root/.cache/go-build \
3635
--mount=type=cache,target=/root/go/pkg/mod \
37-
go build -o /app/wasp -a -tags=${BUILD_TAGS} -ldflags=${BUILD_LD_FLAGS} .
36+
go build -o /app/wasp -a -ldflags=${BUILD_LD_FLAGS} .
3837

3938
############################
4039
# Image

Dockerfile.noncached

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ ARG GOLANG_IMAGE_TAG=1.24-bullseye
33

44
# Build stage
55
FROM golang:${GOLANG_IMAGE_TAG} AS build
6-
ARG BUILD_TAGS=rocksdb
76
ARG BUILD_LD_FLAGS="--X=github.com/iotaledger/wasp/v2/components/app.Version=v0.0.0-testing"
87

98
LABEL org.label-schema.description="Wasp"
@@ -30,7 +29,7 @@ RUN go mod download
3029
# Project build stage
3130
COPY . .
3231

33-
RUN go build -o /app/wasp -a -tags=${BUILD_TAGS} -ldflags=${BUILD_LD_FLAGS} .
32+
RUN go build -o /app/wasp -a -ldflags=${BUILD_LD_FLAGS} .
3433

3534
############################
3635
# Image

Makefile

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
GIT_REF_TAG := $(shell git describe --tags)
2-
BUILD_TAGS = rocksdb
32
BUILD_LD_FLAGS = "-X=github.com/iotaledger/wasp/v2/components/app.Version=$(GIT_REF_TAG)"
43
DOCKER_BUILD_ARGS = # E.g. make docker-build "DOCKER_BUILD_ARGS=--tag wasp:devel"
54

@@ -11,8 +10,8 @@ TEST_PKG=./...
1110
TEST_ARG=
1211

1312
BUILD_PKGS ?= ./
14-
BUILD_CMD=go build -o . -tags $(BUILD_TAGS) -ldflags $(BUILD_LD_FLAGS)
15-
INSTALL_CMD=go install -tags $(BUILD_TAGS) -ldflags $(BUILD_LD_FLAGS)
13+
BUILD_CMD=go build -o . -ldflags $(BUILD_LD_FLAGS)
14+
INSTALL_CMD=go install -ldflags $(BUILD_LD_FLAGS)
1615

1716
# Docker image name and tag
1817
DOCKER_IMAGE_NAME=wasp
@@ -26,7 +25,7 @@ compile-solidity:
2625
cd packages/evm/evmtest/wiki_how_tos && go generate
2726

2827
build-cli:
29-
cd tools/wasp-cli && go mod tidy && go build -ldflags $(BUILD_LD_FLAGS) -tags rocksdb -o ../../
28+
cd tools/wasp-cli && go mod tidy && go build -ldflags $(BUILD_LD_FLAGS) -tags rocksdb -o ../../
3029

3130
build-full: build-cli
3231
$(BUILD_CMD) ./...
@@ -40,16 +39,16 @@ gendoc:
4039
./scripts/gendoc.sh
4140

4241
test-full: install
43-
go test -tags $(BUILD_TAGS),runheavy -race -ldflags $(BUILD_LD_FLAGS) ./... --timeout 60m --count 1 -failfast
42+
go test -tags runheavy -race -ldflags $(BUILD_LD_FLAGS) ./... --timeout 60m --count 1 -failfast
4443

4544
test: install
46-
go test -tags $(BUILD_TAGS) -race -ldflags $(BUILD_LD_FLAGS) $(TEST_PKG) --timeout 90m --count 1 -failfast $(TEST_ARG)
45+
go test -race -ldflags $(BUILD_LD_FLAGS) $(TEST_PKG) --timeout 90m --count 1 -failfast $(TEST_ARG)
4746

4847
test-short:
49-
go test -tags $(BUILD_TAGS) -race -ldflags $(BUILD_LD_FLAGS) --short --count 1 -timeout 25m -failfast $(shell go list ./...)
48+
go test -race -ldflags $(BUILD_LD_FLAGS) --short --count 1 -timeout 25m -failfast $(shell go list ./...)
5049

5150
test-cluster: install
52-
go test -tags $(BUILD_TAGS) -race -ldflags $(BUILD_LD_FLAGS) --count 1 -timeout 25m -failfast $(shell go list ./tools/cluster/tests/...)
51+
go test -race -ldflags $(BUILD_LD_FLAGS) --count 1 -timeout 25m -failfast $(shell go list ./tools/cluster/tests/...)
5352

5453
install-cli:
5554
cd tools/wasp-cli && go mod tidy && go install -ldflags $(BUILD_LD_FLAGS)
@@ -76,7 +75,6 @@ gofumpt-list:
7675

7776
docker-build:
7877
DOCKER_BUILDKIT=1 docker build ${DOCKER_BUILD_ARGS} \
79-
--build-arg BUILD_TAGS=${BUILD_TAGS} \
8078
--build-arg BUILD_LD_FLAGS=${BUILD_LD_FLAGS} \
8179
--tag iotaledger/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) \
8280
.

components/database/component.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func provide(c *dig.Container) error {
9090
database.WithEngine(deps.DatabaseEngine),
9191
database.WithPath(path),
9292
database.WithCacheSize(ParamsDatabase.ChainState.CacheSize),
93+
database.WithBloomFilter(ParamsDatabase.ChainState.BloomFilterBitsPerKey),
9394
)
9495
if err != nil {
9596
Component.LogPanic(err.Error())

components/database/params.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ type ParametersDatabase struct {
1313
// Path defines the path to the chain state databases folder.
1414
Path string `default:"waspdb/chains/data" usage:"the path to the chain state databases folder"`
1515

16-
CacheSize uint64 `default:"33554432" usage:"size of the RocksDB block cache"`
16+
CacheSize uint64 `default:"33554432" usage:"size of the RocksDB block cache"`
17+
BloomFilterBitsPerKey float64 `default:"0" usage:"RocksDB bloom filter bits per key"`
1718
}
1819

1920
// DebugSkipHealthCheck defines whether to ignore the check for corrupted databases.

config_defaults.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
"engine": "rocksdb",
3434
"chainState": {
3535
"path": "waspdb/chains/data",
36-
"cacheSize": 33554432
36+
"cacheSize": 33554432,
37+
"bloomFilterBitsPerKey": 0
3738
},
3839
"debugSkipHealthCheck": true,
3940
"readOnlyFilePath": ""

documentation/docs/configuration.md

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,11 @@ Example:
150150

151151
### <a id="db_chainstate"></a> ChainState
152152

153-
| Name | Description | Type | Default value |
154-
| --------- | -------------------------------------------- | ------ | -------------------- |
155-
| path | The path to the chain state databases folder | string | "waspdb/chains/data" |
156-
| cacheSize | Size of the RocksDB block cache | uint | 33554432 |
153+
| Name | Description | Type | Default value |
154+
| --------------------- | -------------------------------------------- | ------ | -------------------- |
155+
| path | The path to the chain state databases folder | string | "waspdb/chains/data" |
156+
| cacheSize | Size of the RocksDB block cache | uint | 33554432 |
157+
| bloomFilterBitsPerKey | RocksDB bloom filter bits per key | float | 0.0 |
157158

158159
Example:
159160

@@ -163,7 +164,8 @@ Example:
163164
"engine": "rocksdb",
164165
"chainState": {
165166
"path": "waspdb/chains/data",
166-
"cacheSize": 33554432
167+
"cacheSize": 33554432,
168+
"bloomFilterBitsPerKey": 0
167169
},
168170
"debugSkipHealthCheck": true,
169171
"readOnlyFilePath": ""

packages/database/database.go

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"github.com/iotaledger/hive.go/runtime/ioutils"
1111
"github.com/iotaledger/wasp/v2/packages/chaindb"
1212
"github.com/iotaledger/wasp/v2/packages/kvstore"
13-
"github.com/iotaledger/wasp/v2/packages/kvstore/rocksdb"
1413
)
1514

1615
var AllowedEngines = []hivedb.Engine{
@@ -89,7 +88,7 @@ func CheckEngine(dbPath string, createDatabaseIfNotExists bool, dbEngine hivedb.
8988
}
9089

9190
func NewDatabaseInMemory() (*Database, error) {
92-
return NewDatabase(hivedb.EngineMapDB, "", false, 0)
91+
return newDatabaseMapDB(), nil
9392
}
9493

9594
// NewDatabase opens a database.
@@ -99,6 +98,7 @@ func NewDatabase(
9998
path string,
10099
createDatabaseIfNotExists bool,
101100
cacheSize uint64,
101+
bloomFilterBitsPerKey float64,
102102
) (*Database, error) {
103103
targetEngine, err := CheckEngine(path, createDatabaseIfNotExists, dbEngine)
104104
if err != nil {
@@ -107,7 +107,11 @@ func NewDatabase(
107107

108108
switch targetEngine {
109109
case hivedb.EngineRocksDB:
110-
return newDatabaseRocksDB(path, cacheSize)
110+
return newDatabaseRocksDB(
111+
path,
112+
cacheSize,
113+
bloomFilterBitsPerKey,
114+
)
111115

112116
case hivedb.EngineMapDB:
113117
return newDatabaseMapDB(), nil
@@ -117,20 +121,6 @@ func NewDatabase(
117121
}
118122
}
119123

120-
func NewReadOnlyDatabase(
121-
path string,
122-
) (*Database, error) {
123-
dbConn, err := rocksdb.OpenDBReadOnly(path)
124-
if err != nil {
125-
return nil, fmt.Errorf("failed to open read-only RocksDB: %w", err)
126-
}
127-
128-
db := New(path, rocksdb.New(dbConn), hivedb.EngineRocksDB, false, func() bool {
129-
return false
130-
})
131-
return db, nil
132-
}
133-
134124
type databaseWithHealthTracker struct {
135125
database *Database
136126
storeHealthTracker *kvstore.StoreHealthTracker
@@ -140,10 +130,11 @@ func newDatabaseWithHealthTracker(
140130
path string,
141131
dbEngine hivedb.Engine,
142132
cacheSize uint64,
133+
bloomFilterBitsPerKey float64,
143134
storeVersion byte,
144135
storeVersionUpdateFunc StoreVersionUpdateFunc,
145136
) (*databaseWithHealthTracker, error) {
146-
db, err := NewDatabase(dbEngine, path, true, cacheSize)
137+
db, err := NewDatabase(dbEngine, path, true, cacheSize, bloomFilterBitsPerKey)
147138
if err != nil {
148139
return nil, err
149140
}

packages/database/manager.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ type ChainStateDatabaseManager struct {
2626
mutex sync.RWMutex
2727

2828
// options
29-
engine hivedb.Engine
30-
databasePath string
31-
cacheSize uint64
29+
engine hivedb.Engine
30+
databasePath string
31+
cacheSize uint64
32+
bloomFilterBitsPerKey float64
3233

3334
// databases
3435
databases map[isc.ChainID]*databaseWithHealthTracker
@@ -52,6 +53,12 @@ func WithCacheSize(cacheSize uint64) options.Option[ChainStateDatabaseManager] {
5253
}
5354
}
5455

56+
func WithBloomFilter(bitsPerKey float64) options.Option[ChainStateDatabaseManager] {
57+
return func(d *ChainStateDatabaseManager) {
58+
d.bloomFilterBitsPerKey = bitsPerKey
59+
}
60+
}
61+
5562
func NewChainStateDatabaseManager(chainRecordRegistryProvider registry.ChainRecordRegistryProvider, opts ...options.Option[ChainStateDatabaseManager]) (*ChainStateDatabaseManager, error) {
5663
m := options.Apply(&ChainStateDatabaseManager{
5764
engine: hivedb.EngineAuto,
@@ -103,6 +110,7 @@ func (m *ChainStateDatabaseManager) createDatabase(chainID isc.ChainID) (*databa
103110
path.Join(m.databasePath, chainID.String()),
104111
m.engine,
105112
m.cacheSize,
113+
m.bloomFilterBitsPerKey,
106114
StoreVersionChainState,
107115
nil,
108116
)

0 commit comments

Comments
 (0)