Skip to content

Cronos 1.5.1 node does not reach the required block height #1905

@P3TP0

Description

@P3TP0

I have two absolutely identical Cronos 1.5.1 nodes, their configurations are absolutely identical. Moreover, the hardware on the two nodes is also identical, located in the same datacenter. That is, the conditions for the existence of these nodes are absolutely the same.

Before upgrading to version 1.5.1, both nodes were working properly and without problems. However, as soon as I upgraded them to version 1.5.1, the first node continued to work normally at the same height as the block explorer, and the second node formed a block height drop.

At the same time, both nodes have the same step and timings for writing new blocks, but there is a gap between them.

For example:
first node = 39954716
second node (problematic) = 39888291

The difference between them is about 66425 blocks.

There are no errors or warnings in the logs in normal mode and in debug mode, as if everything is in order. At the same time, the logs in debug mode show that the problematic second node sees the current height:

12:15PM DBG Receive chId=32 module=consensus msg="[NewRoundStep H:40017682 R:0S:RoundStepNewHeight LCR:0]" server=node src="Peer{MConn{134.65.193.137:26656}648894895a2c0589303ab464a2f8b197b2ac746b out}"
12:15PM DBG Read PacketMsg conn=MConn{134.65.193.137:26656} module=p2p packet={"eof":true,"data":"OgkIkb6KExgCIBo=","channel_id":32} [email protected]:26656 server=node

Next, I attach the configuration files, which are identical for both nodes:

config.toml

proxy_app = "tcp://127.0.0.1:26658"
moniker = "cronos-node"
fast_sync = true

db_backend = "rocksdb"
db_dir = "/opt/cronos/data"

log_level = "info"
log_format = "plain"

genesis_file = "/opt/cronos/config/genesis.json"
priv_validator_key_file = "/opt/cronos/config/priv_validator_key.json"
priv_validator_state_file = "/opt/cronos/config/priv_validator_state.json"
node_key_file = "/opt/cronos/config/node_key.json"

abci = "socket"
filter_peers = false

[rpc]
laddr = "tcp://0.0.0.0:26657"
cors_allowed_origins = []
cors_allowed_methods = ["HEAD", "GET", "POST", ]
cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
grpc_laddr = ""
grpc_max_open_connections = 900
unsafe = false
max_open_connections = 900
max_subscription_clients = 100
max_subscriptions_per_client = 5
experimental_subscription_buffer_size = 200
experimental_websocket_write_buffer_size = 200
experimental_close_on_slow_client = false
timeout_broadcast_tx_commit = "20s"
max_body_bytes = 1000000
max_header_bytes = 1048576
tls_cert_file = ""
tls_key_file = ""
pprof_laddr = "0.0.0.0:6060"

[p2p]
laddr = "tcp://0.0.0.0:26656"
external_address = ""
seeds = "0d5cf1394a1cfde28dc8f023567222abc0f47534@cronos-seed-0.crypto.org:26656,3032073adc06d710dd512240281637c1bd0c8a7b@cronos-seed-1.crypto.org:26656,04f43116b4c6c70054d9c2b7485383df5b1ed1da@cronos-seed-2.crypto.org:26656,337377dcda43d79c537d2c4d93ad3b698ce9452e@bd-cronos-mainnet-seed-node-01.bdnodes.net:26656"
persistent_peers = ""
upnp = false
addr_book_file = "/opt/cronos/config/addrbook.json"
addr_book_strict = true
max_num_inbound_peers = 30
max_num_outbound_peers = 20
unconditional_peer_ids = ""
persistent_peers_max_dial_period = "0s"
flush_throttle_timeout = "100ms"
max_packet_msg_payload_size = 1024
send_rate = 51200000
recv_rate = 51200000
pex = true
seed_mode = false
private_peer_ids = ""
allow_duplicate_ip = false
handshake_timeout = "20s"
dial_timeout = "3s"

[mempool]
version = "v1"
recheck = true
broadcast = true
wal_dir = ""
size = 2000
max_txs_bytes = 1073741824
cache_size = 10000
keep-invalid-txs-in-cache = false
max_tx_bytes = 1048576
max_batch_bytes = 0
ttl-duration = "0s"
ttl-num-blocks = 0

[statesync]
enable = "false"
rpc_servers = ","
trust_height = 0
trust_hash = ""
trust_period = "168h0m0s"
discovery_time = "15s"
temp_dir = ""
chunk_request_timeout = "10s"
chunk_fetchers = "4"

[fastsync]
version = "v0"

[consensus]
wal_file = "data/cs.wal/wal"
timeout_propose = "3s"
timeout_propose_delta = "500ms"
timeout_prevote = "1s"
timeout_prevote_delta = "500ms"
timeout_precommit = "1s"
timeout_precommit_delta = "500ms"
timeout_commit = "5s"
double_sign_check_height = 0
skip_timeout_commit = false
create_empty_blocks = true
create_empty_blocks_interval = "5s"
peer_gossip_sleep_duration = "100ms"
peer_query_maj23_sleep_duration = "2s"

[storage]
discard_abci_responses = false

[tx_index]
indexer = "kv"
psql-conn = ""

[instrumentation]
prometheus = true
prometheus_listen_addr = ":26660"
max_open_connections = 3
namespace = "tendermint"

app.toml

minimum-gas-prices = "5000000000000basecro"
query-gas-limit = "100000000"
pruning = "default"
pruning-keep-recent = "0"
pruning-keep-every = "0"
pruning-interval = "0"
halt-height = 0
halt-time = 0
min-retain-blocks = 0
inter-block-cache = true
index-events = []
iavl-cache-size = 781250
iavl-disable-fastnode = false
app-db-backend = "rocksdb"

[telemetry]
service-name = ""
enabled = true
enable-hostname = false
enable-hostname-label = false
enable-service-label = false
prometheus-retention-time = 0

[api]
enable = true
swagger = true
address = "tcp://0.0.0.0:1317"
max-open-connections = 1000
rpc-read-timeout = 10
rpc-write-timeout = 0
rpc-max-body-bytes = 1000000
enabled-unsafe-cors = true

[rosetta]
enable = false
address = ":8080"
blockchain = "app"
network = "network"
retries = 3
offline = false
enable-fee-suggestion = false
gas-to-suggest = 200000
denom-to-suggest = "basecro"

[grpc]
enable = true
address = "0.0.0.0:9090"
max-recv-msg-size = "10485760"
max-send-msg-size = "2147483647"

[grpc-web]
enable = true
address = "0.0.0.0:9091"
enable-unsafe-cors = false

[state-sync]
snapshot-interval = 0
snapshot-keep-recent = 10

[store]
streamers = []

[streamers]
[streamers.file]
keys = ["*", ]
write_dir = ""
prefix = ""
output-metadata = "true"
stop-node-on-error = "true"
fsync = "false"

[evm]
tracer = ""
max-tx-gas-wanted = 50000

[json-rpc]
enable = true
address = "0.0.0.0:8545"
ws-address = "0.0.0.0:8546"
api = "eth,net,web3,txpool,personal"
gas-cap = 25000000
evm-timeout = "60s"
txfee-cap = 1
filter-cap = 200
feehistory-cap = 100
logs-cap = 10000
block-range-cap = 2000
http-timeout = "60s"
http-idle-timeout = "120s"
allow-unprotected-txs = false
max-open-connections = 0
enable-indexer = false
metrics-address = "0.0.0.0:6065"

[tls]
certificate-path = ""
key-path = ""

client.toml

chain-id = "cronosmainnet_25-1"
keyring-backend = "os"
output = "text"
node = "tcp://0.0.0.0:8545"
broadcast-mode = "sync"

I would appreciate any possible help!

Red - 1st node
Blue - 2nd node (problematic)

Image Image

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions