-
Notifications
You must be signed in to change notification settings - Fork 274
Description
I have two absolutely identical Cronos 1.5.1 nodes, their configurations are absolutely identical. Moreover, the hardware on the two nodes is also identical, located in the same datacenter. That is, the conditions for the existence of these nodes are absolutely the same.
Before upgrading to version 1.5.1, both nodes were working properly and without problems. However, as soon as I upgraded them to version 1.5.1, the first node continued to work normally at the same height as the block explorer, and the second node formed a block height drop.
At the same time, both nodes have the same step and timings for writing new blocks, but there is a gap between them.
For example:
first node = 39954716
second node (problematic) = 39888291
The difference between them is about 66425 blocks.
There are no errors or warnings in the logs in normal mode and in debug mode, as if everything is in order. At the same time, the logs in debug mode show that the problematic second node sees the current height:
12:15PM DBG Receive chId=32 module=consensus msg="[NewRoundStep H:40017682 R:0S:RoundStepNewHeight LCR:0]" server=node src="Peer{MConn{134.65.193.137:26656}648894895a2c0589303ab464a2f8b197b2ac746b out}"
12:15PM DBG Read PacketMsg conn=MConn{134.65.193.137:26656} module=p2p packet={"eof":true,"data":"OgkIkb6KExgCIBo=","channel_id":32} [email protected]:26656 server=node
Next, I attach the configuration files, which are identical for both nodes:
config.toml
proxy_app = "tcp://127.0.0.1:26658"
moniker = "cronos-node"
fast_sync = true
db_backend = "rocksdb"
db_dir = "/opt/cronos/data"
log_level = "info"
log_format = "plain"
genesis_file = "/opt/cronos/config/genesis.json"
priv_validator_key_file = "/opt/cronos/config/priv_validator_key.json"
priv_validator_state_file = "/opt/cronos/config/priv_validator_state.json"
node_key_file = "/opt/cronos/config/node_key.json"
abci = "socket"
filter_peers = false
[rpc]
laddr = "tcp://0.0.0.0:26657"
cors_allowed_origins = []
cors_allowed_methods = ["HEAD", "GET", "POST", ]
cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
grpc_laddr = ""
grpc_max_open_connections = 900
unsafe = false
max_open_connections = 900
max_subscription_clients = 100
max_subscriptions_per_client = 5
experimental_subscription_buffer_size = 200
experimental_websocket_write_buffer_size = 200
experimental_close_on_slow_client = false
timeout_broadcast_tx_commit = "20s"
max_body_bytes = 1000000
max_header_bytes = 1048576
tls_cert_file = ""
tls_key_file = ""
pprof_laddr = "0.0.0.0:6060"
[p2p]
laddr = "tcp://0.0.0.0:26656"
external_address = ""
seeds = "0d5cf1394a1cfde28dc8f023567222abc0f47534@cronos-seed-0.crypto.org:26656,3032073adc06d710dd512240281637c1bd0c8a7b@cronos-seed-1.crypto.org:26656,04f43116b4c6c70054d9c2b7485383df5b1ed1da@cronos-seed-2.crypto.org:26656,337377dcda43d79c537d2c4d93ad3b698ce9452e@bd-cronos-mainnet-seed-node-01.bdnodes.net:26656"
persistent_peers = ""
upnp = false
addr_book_file = "/opt/cronos/config/addrbook.json"
addr_book_strict = true
max_num_inbound_peers = 30
max_num_outbound_peers = 20
unconditional_peer_ids = ""
persistent_peers_max_dial_period = "0s"
flush_throttle_timeout = "100ms"
max_packet_msg_payload_size = 1024
send_rate = 51200000
recv_rate = 51200000
pex = true
seed_mode = false
private_peer_ids = ""
allow_duplicate_ip = false
handshake_timeout = "20s"
dial_timeout = "3s"
[mempool]
version = "v1"
recheck = true
broadcast = true
wal_dir = ""
size = 2000
max_txs_bytes = 1073741824
cache_size = 10000
keep-invalid-txs-in-cache = false
max_tx_bytes = 1048576
max_batch_bytes = 0
ttl-duration = "0s"
ttl-num-blocks = 0
[statesync]
enable = "false"
rpc_servers = ","
trust_height = 0
trust_hash = ""
trust_period = "168h0m0s"
discovery_time = "15s"
temp_dir = ""
chunk_request_timeout = "10s"
chunk_fetchers = "4"
[fastsync]
version = "v0"
[consensus]
wal_file = "data/cs.wal/wal"
timeout_propose = "3s"
timeout_propose_delta = "500ms"
timeout_prevote = "1s"
timeout_prevote_delta = "500ms"
timeout_precommit = "1s"
timeout_precommit_delta = "500ms"
timeout_commit = "5s"
double_sign_check_height = 0
skip_timeout_commit = false
create_empty_blocks = true
create_empty_blocks_interval = "5s"
peer_gossip_sleep_duration = "100ms"
peer_query_maj23_sleep_duration = "2s"
[storage]
discard_abci_responses = false
[tx_index]
indexer = "kv"
psql-conn = ""
[instrumentation]
prometheus = true
prometheus_listen_addr = ":26660"
max_open_connections = 3
namespace = "tendermint"
app.toml
minimum-gas-prices = "5000000000000basecro"
query-gas-limit = "100000000"
pruning = "default"
pruning-keep-recent = "0"
pruning-keep-every = "0"
pruning-interval = "0"
halt-height = 0
halt-time = 0
min-retain-blocks = 0
inter-block-cache = true
index-events = []
iavl-cache-size = 781250
iavl-disable-fastnode = false
app-db-backend = "rocksdb"
[telemetry]
service-name = ""
enabled = true
enable-hostname = false
enable-hostname-label = false
enable-service-label = false
prometheus-retention-time = 0
[api]
enable = true
swagger = true
address = "tcp://0.0.0.0:1317"
max-open-connections = 1000
rpc-read-timeout = 10
rpc-write-timeout = 0
rpc-max-body-bytes = 1000000
enabled-unsafe-cors = true
[rosetta]
enable = false
address = ":8080"
blockchain = "app"
network = "network"
retries = 3
offline = false
enable-fee-suggestion = false
gas-to-suggest = 200000
denom-to-suggest = "basecro"
[grpc]
enable = true
address = "0.0.0.0:9090"
max-recv-msg-size = "10485760"
max-send-msg-size = "2147483647"
[grpc-web]
enable = true
address = "0.0.0.0:9091"
enable-unsafe-cors = false
[state-sync]
snapshot-interval = 0
snapshot-keep-recent = 10
[store]
streamers = []
[streamers]
[streamers.file]
keys = ["*", ]
write_dir = ""
prefix = ""
output-metadata = "true"
stop-node-on-error = "true"
fsync = "false"
[evm]
tracer = ""
max-tx-gas-wanted = 50000
[json-rpc]
enable = true
address = "0.0.0.0:8545"
ws-address = "0.0.0.0:8546"
api = "eth,net,web3,txpool,personal"
gas-cap = 25000000
evm-timeout = "60s"
txfee-cap = 1
filter-cap = 200
feehistory-cap = 100
logs-cap = 10000
block-range-cap = 2000
http-timeout = "60s"
http-idle-timeout = "120s"
allow-unprotected-txs = false
max-open-connections = 0
enable-indexer = false
metrics-address = "0.0.0.0:6065"
[tls]
certificate-path = ""
key-path = ""
client.toml
chain-id = "cronosmainnet_25-1"
keyring-backend = "os"
output = "text"
node = "tcp://0.0.0.0:8545"
broadcast-mode = "sync"
I would appreciate any possible help!
Red - 1st node
Blue - 2nd node (problematic)
