From 880fb2438deba18758d2155ce6abc6f1d390193a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Mon, 10 Feb 2025 13:25:06 +0300 Subject: [PATCH 01/12] feat: add migration checker cmd --- cmd/migration-checker/main.go | 241 ++++++++++++++++++++++++++++++++++ trie/secure_trie.go | 10 ++ trie/tracer.go | 24 ++++ trie/zk_trie.go | 39 ++++++ 4 files changed, 314 insertions(+) create mode 100644 cmd/migration-checker/main.go diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go new file mode 100644 index 00000000000..9a01b04d841 --- /dev/null +++ b/cmd/migration-checker/main.go @@ -0,0 +1,241 @@ +package main + +import ( + "bytes" + "encoding/hex" + "flag" + "fmt" + "os" + "runtime" + "sync" + "sync/atomic" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/crypto" + "github.com/scroll-tech/go-ethereum/ethdb/leveldb" + "github.com/scroll-tech/go-ethereum/rlp" + "github.com/scroll-tech/go-ethereum/trie" +) + +var accountsDone atomic.Uint64 +var trieCheckers = make(chan struct{}, runtime.GOMAXPROCS(0)*4) + +type dbs struct { + zkDb *leveldb.Database + mptDb *leveldb.Database +} + +func main() { + var ( + mptDbPath = flag.String("mpt-db", "", "path to the MPT node DB") + zkDbPath = flag.String("zk-db", "", "path to the ZK node DB") + mptRoot = flag.String("mpt-root", "", "root hash of the MPT node") + zkRoot = flag.String("zk-root", "", "root hash of the ZK node") + ) + flag.Parse() + + zkDb, err := leveldb.New(*zkDbPath, 1024, 128, "", true) + panicOnError(err, "", "failed to open zk db") + mptDb, err := leveldb.New(*mptDbPath, 1024, 128, "", true) + panicOnError(err, "", "failed to open mpt db") + + zkRootHash := common.HexToHash(*zkRoot) + mptRootHash := common.HexToHash(*mptRoot) + + for i := 0; i < runtime.GOMAXPROCS(0)*4; i++ { + trieCheckers <- struct{}{} + } + + checkTrieEquality(&dbs{ + zkDb: zkDb, + mptDb: mptDb, + }, zkRootHash, mptRootHash, "", checkAccountEquality, true) +} + +func panicOnError(err error, label, msg string) { + if err != nil { + panic(fmt.Sprint(label, " error: ", msg, " ", err)) + } +} + +func dup(s []byte) []byte { + return append([]byte{}, s...) +} +func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte), top bool) { + zkTrie, err := trie.NewZkTrie(zkRoot, trie.NewZktrieDatabaseFromTriedb(trie.NewDatabaseWithConfig(dbs.zkDb, &trie.Config{Preimages: true}))) + panicOnError(err, label, "failed to create zk trie") + mptTrie, err := trie.NewSecureNoTracer(mptRoot, trie.NewDatabaseWithConfig(dbs.mptDb, &trie.Config{Preimages: true})) + panicOnError(err, label, "failed to create mpt trie") + + mptLeafCh := loadMPT(mptTrie, top) + zkLeafCh := loadZkTrie(zkTrie, top) + + mptLeafMap := <-mptLeafCh + zkLeafMap := <-zkLeafCh + + if len(mptLeafMap) != len(zkLeafMap) { + panic(fmt.Sprintf("%s MPT and ZK trie leaf count mismatch: MPT: %d, ZK: %d", label, len(mptLeafMap), len(zkLeafMap))) + } + + for preimageKey, zkValue := range zkLeafMap { + if top { + // ZkTrie pads preimages with 0s to make them 32 bytes. + // So we might need to clear those zeroes here since we need 20 byte addresses at top level (ie state trie) + if len(preimageKey) > 20 { + for _, b := range []byte(preimageKey)[20:] { + if b != 0 { + panic(fmt.Sprintf("%s padded byte is not 0 (preimage %s)", label, hex.EncodeToString([]byte(preimageKey)))) + } + } + preimageKey = preimageKey[:20] + } + } else if len(preimageKey) != 32 { + // storage leafs should have 32 byte keys, pad them if needed + zeroes := make([]byte, 32) + copy(zeroes, []byte(preimageKey)) + preimageKey = string(zeroes) + } + + mptKey := crypto.Keccak256([]byte(preimageKey)) + mptVal, ok := mptLeafMap[string(mptKey)] + if !ok { + panic(fmt.Sprintf("%s key %s (preimage %s) not found in mpt", label, hex.EncodeToString([]byte(mptKey)), hex.EncodeToString([]byte(preimageKey)))) + } + + leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(preimageKey))), dbs, zkValue, mptVal) + } +} + +func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountBytes []byte) { + mptAccount := &types.StateAccount{} + panicOnError(rlp.DecodeBytes(mptAccountBytes, mptAccount), label, "failed to decode mpt account") + zkAccount, err := types.UnmarshalStateAccount(zkAccountBytes) + panicOnError(err, label, "failed to decode zk account") + + if mptAccount.Nonce != zkAccount.Nonce { + panic(fmt.Sprintf("%s nonce mismatch: zk: %d, mpt: %d", label, zkAccount.Nonce, mptAccount.Nonce)) + } + + if mptAccount.Balance.Cmp(zkAccount.Balance) != 0 { + panic(fmt.Sprintf("%s balance mismatch: zk: %s, mpt: %s", label, zkAccount.Balance.String(), mptAccount.Balance.String())) + } + + if !bytes.Equal(mptAccount.KeccakCodeHash, zkAccount.KeccakCodeHash) { + panic(fmt.Sprintf("%s code hash mismatch: zk: %s, mpt: %s", label, hex.EncodeToString(zkAccount.KeccakCodeHash), hex.EncodeToString(mptAccount.KeccakCodeHash))) + } + + if (zkAccount.Root == common.Hash{}) != (mptAccount.Root == types.EmptyRootHash) { + panic(fmt.Sprintf("%s empty account root mismatch", label)) + } else if zkAccount.Root != (common.Hash{}) { + zkRoot := common.BytesToHash(zkAccount.Root[:]) + mptRoot := common.BytesToHash(mptAccount.Root[:]) + <-trieCheckers + go func() { + defer func() { + if p := recover(); p != nil { + fmt.Println(p) + os.Exit(1) + } + }() + + checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false) + accountsDone.Add(1) + fmt.Println("Accounts done:", accountsDone.Load()) + trieCheckers <- struct{}{} + }() + } else { + accountsDone.Add(1) + fmt.Println("Accounts done:", accountsDone.Load()) + } +} + +func checkStorageEquality(label string, _ *dbs, zkStorageBytes, mptStorageBytes []byte) { + zkValue := common.BytesToHash(zkStorageBytes) + _, content, _, err := rlp.Split(mptStorageBytes) + panicOnError(err, label, "failed to decode mpt storage") + mptValue := common.BytesToHash(content) + if !bytes.Equal(zkValue[:], mptValue[:]) { + panic(fmt.Sprintf("%s storage mismatch: zk: %s, mpt: %s", label, zkValue.Hex(), mptValue.Hex())) + } +} + +func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { + startKey := make([]byte, 32) + workers := 1 << 5 + if !parallel { + workers = 1 + } + step := byte(0xFF) / byte(workers) + + mptLeafMap := make(map[string][]byte, 1000) + var mptLeafMutex sync.Mutex + + var mptWg sync.WaitGroup + for i := 0; i < workers; i++ { + startKey[0] = byte(i) * step + trieIt := trie.NewIterator(mptTrie.NodeIterator(startKey)) + + mptWg.Add(1) + go func() { + defer mptWg.Done() + for trieIt.Next() { + if parallel { + mptLeafMutex.Lock() + } + + if _, ok := mptLeafMap[string(trieIt.Key)]; ok { + mptLeafMutex.Unlock() + break + } + + mptLeafMap[string(dup(trieIt.Key))] = dup(trieIt.Value) + + if parallel { + mptLeafMutex.Unlock() + } + + if parallel && len(mptLeafMap)%10000 == 0 { + fmt.Println("MPT Accounts Loaded:", len(mptLeafMap)) + } + } + }() + } + + respChan := make(chan map[string][]byte) + go func() { + mptWg.Wait() + respChan <- mptLeafMap + }() + return respChan +} + +func loadZkTrie(zkTrie *trie.ZkTrie, parallel bool) chan map[string][]byte { + zkLeafMap := make(map[string][]byte, 1000) + var zkLeafMutex sync.Mutex + zkDone := make(chan map[string][]byte) + go func() { + zkTrie.CountLeaves(func(key, value []byte) { + preimageKey := zkTrie.GetKey(key) + if len(preimageKey) == 0 { + panic(fmt.Sprintf("preimage not found zk trie %s", hex.EncodeToString(key))) + } + + if parallel { + zkLeafMutex.Lock() + } + + zkLeafMap[string(dup(preimageKey))] = value + + if parallel { + zkLeafMutex.Unlock() + } + + if parallel && len(zkLeafMap)%10000 == 0 { + fmt.Println("ZK Accounts Loaded:", len(zkLeafMap)) + } + }, parallel) + zkDone <- zkLeafMap + }() + return zkDone +} diff --git a/trie/secure_trie.go b/trie/secure_trie.go index a3529c2fc68..90de589b403 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -65,6 +65,16 @@ func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) { return &SecureTrie{trie: *trie, preimages: db.preimages}, nil } +func NewSecureNoTracer(root common.Hash, db *Database) (*SecureTrie, error) { + t, err := NewSecure(root, db) + if err != nil { + return nil, err + } + + t.trie.tracer = nil + return t, nil +} + // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. func (t *SecureTrie) Get(key []byte) []byte { diff --git a/trie/tracer.go b/trie/tracer.go index 99cda0706f7..f517195eff8 100644 --- a/trie/tracer.go +++ b/trie/tracer.go @@ -60,6 +60,10 @@ func newTracer() *tracer { // blob internally. Don't change the value outside of function since // it's not deep-copied. func (t *tracer) onRead(path []byte, val []byte) { + if t == nil { + return + } + t.accessList[string(path)] = val } @@ -67,6 +71,10 @@ func (t *tracer) onRead(path []byte, val []byte) { // in the deletion set (resurrected node), then just wipe it from // the deletion set as it's "untouched". func (t *tracer) onInsert(path []byte) { + if t == nil { + return + } + if _, present := t.deletes[string(path)]; present { delete(t.deletes, string(path)) return @@ -78,6 +86,10 @@ func (t *tracer) onInsert(path []byte) { // in the addition set, then just wipe it from the addition set // as it's untouched. func (t *tracer) onDelete(path []byte) { + if t == nil { + return + } + if _, present := t.inserts[string(path)]; present { delete(t.inserts, string(path)) return @@ -87,6 +99,10 @@ func (t *tracer) onDelete(path []byte) { // reset clears the content tracked by tracer. func (t *tracer) reset() { + if t == nil { + return + } + t.inserts = make(map[string]struct{}) t.deletes = make(map[string]struct{}) t.accessList = make(map[string][]byte) @@ -94,6 +110,10 @@ func (t *tracer) reset() { // copy returns a deep copied tracer instance. func (t *tracer) copy() *tracer { + if t == nil { + return nil + } + accessList := make(map[string][]byte, len(t.accessList)) for path, blob := range t.accessList { accessList[path] = common.CopyBytes(blob) @@ -107,6 +127,10 @@ func (t *tracer) copy() *tracer { // deletedNodes returns a list of node paths which are deleted from the trie. func (t *tracer) deletedNodes() []string { + if t == nil { + return nil + } + var paths []string for path := range t.deletes { // It's possible a few deleted nodes were embedded diff --git a/trie/zk_trie.go b/trie/zk_trie.go index af2dfc3bc04..57fac5b8040 100644 --- a/trie/zk_trie.go +++ b/trie/zk_trie.go @@ -238,3 +238,42 @@ func VerifyProofSMT(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueRead func (t *ZkTrie) Witness() map[string]struct{} { panic("not implemented") } + +func (t *ZkTrie) CountLeaves(cb func(key, value []byte), parallel bool) uint64 { + root, err := t.ZkTrie.Tree().Root() + if err != nil { + panic("CountLeaves cannot get root") + } + return t.countLeaves(root, cb, 0, parallel) +} + +func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth int, parallel bool) uint64 { + if root == nil { + return 0 + } + + rootNode, err := t.ZkTrie.Tree().GetNode(root) + if err != nil { + panic("countLeaves cannot get rootNode") + } + + if rootNode.Type == zktrie.NodeTypeLeaf_New { + cb(append([]byte{}, rootNode.NodeKey.Bytes()...), append([]byte{}, rootNode.Data()...)) + return 1 + } else { + if parallel && depth < 5 { + count := make(chan uint64) + leftT := t.Copy() + rightT := t.Copy() + go func() { + count <- leftT.countLeaves(rootNode.ChildL, cb, depth+1, parallel) + }() + go func() { + count <- rightT.countLeaves(rootNode.ChildR, cb, depth+1, parallel) + }() + return <-count + <-count + } else { + return t.countLeaves(rootNode.ChildL, cb, depth+1, parallel) + t.countLeaves(rootNode.ChildR, cb, depth+1, parallel) + } + } +} From 4f42703927d626f48a6040e51a68b15a88e3332b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Mon, 17 Feb 2025 14:33:35 +0300 Subject: [PATCH 02/12] wait for all checkers to finish --- cmd/migration-checker/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 9a01b04d841..4e1d971092d 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -51,6 +51,10 @@ func main() { zkDb: zkDb, mptDb: mptDb, }, zkRootHash, mptRootHash, "", checkAccountEquality, true) + + for i := 0; i < runtime.GOMAXPROCS(0)*4; i++ { + <-trieCheckers + } } func panicOnError(err error, label, msg string) { From 12ec3a6fe3cd539d4bd91e228219f2f1ca833303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Thu, 20 Feb 2025 18:11:29 +0300 Subject: [PATCH 03/12] paranoid flag --- cmd/migration-checker/main.go | 19 ++++++++++--------- trie/zk_trie.go | 22 ++++++++++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 4e1d971092d..febfba1eea2 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -32,6 +32,7 @@ func main() { zkDbPath = flag.String("zk-db", "", "path to the ZK node DB") mptRoot = flag.String("mpt-root", "", "root hash of the MPT node") zkRoot = flag.String("zk-root", "", "root hash of the ZK node") + paranoid = flag.Bool("paranoid", false, "verifies all node contents against their expected hash") ) flag.Parse() @@ -50,7 +51,7 @@ func main() { checkTrieEquality(&dbs{ zkDb: zkDb, mptDb: mptDb, - }, zkRootHash, mptRootHash, "", checkAccountEquality, true) + }, zkRootHash, mptRootHash, "", checkAccountEquality, true, *paranoid) for i := 0; i < runtime.GOMAXPROCS(0)*4; i++ { <-trieCheckers @@ -66,14 +67,14 @@ func panicOnError(err error, label, msg string) { func dup(s []byte) []byte { return append([]byte{}, s...) } -func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte), top bool) { +func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte, bool), top, paranoid bool) { zkTrie, err := trie.NewZkTrie(zkRoot, trie.NewZktrieDatabaseFromTriedb(trie.NewDatabaseWithConfig(dbs.zkDb, &trie.Config{Preimages: true}))) panicOnError(err, label, "failed to create zk trie") mptTrie, err := trie.NewSecureNoTracer(mptRoot, trie.NewDatabaseWithConfig(dbs.mptDb, &trie.Config{Preimages: true})) panicOnError(err, label, "failed to create mpt trie") mptLeafCh := loadMPT(mptTrie, top) - zkLeafCh := loadZkTrie(zkTrie, top) + zkLeafCh := loadZkTrie(zkTrie, top, paranoid) mptLeafMap := <-mptLeafCh zkLeafMap := <-zkLeafCh @@ -107,11 +108,11 @@ func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leaf panic(fmt.Sprintf("%s key %s (preimage %s) not found in mpt", label, hex.EncodeToString([]byte(mptKey)), hex.EncodeToString([]byte(preimageKey)))) } - leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(preimageKey))), dbs, zkValue, mptVal) + leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(preimageKey))), dbs, zkValue, mptVal, paranoid) } } -func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountBytes []byte) { +func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountBytes []byte, paranoid bool) { mptAccount := &types.StateAccount{} panicOnError(rlp.DecodeBytes(mptAccountBytes, mptAccount), label, "failed to decode mpt account") zkAccount, err := types.UnmarshalStateAccount(zkAccountBytes) @@ -143,7 +144,7 @@ func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountByte } }() - checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false) + checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false, paranoid) accountsDone.Add(1) fmt.Println("Accounts done:", accountsDone.Load()) trieCheckers <- struct{}{} @@ -154,7 +155,7 @@ func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountByte } } -func checkStorageEquality(label string, _ *dbs, zkStorageBytes, mptStorageBytes []byte) { +func checkStorageEquality(label string, _ *dbs, zkStorageBytes, mptStorageBytes []byte, _ bool) { zkValue := common.BytesToHash(zkStorageBytes) _, content, _, err := rlp.Split(mptStorageBytes) panicOnError(err, label, "failed to decode mpt storage") @@ -214,7 +215,7 @@ func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { return respChan } -func loadZkTrie(zkTrie *trie.ZkTrie, parallel bool) chan map[string][]byte { +func loadZkTrie(zkTrie *trie.ZkTrie, parallel, paranoid bool) chan map[string][]byte { zkLeafMap := make(map[string][]byte, 1000) var zkLeafMutex sync.Mutex zkDone := make(chan map[string][]byte) @@ -238,7 +239,7 @@ func loadZkTrie(zkTrie *trie.ZkTrie, parallel bool) chan map[string][]byte { if parallel && len(zkLeafMap)%10000 == 0 { fmt.Println("ZK Accounts Loaded:", len(zkLeafMap)) } - }, parallel) + }, parallel, paranoid) zkDone <- zkLeafMap }() return zkDone diff --git a/trie/zk_trie.go b/trie/zk_trie.go index 57fac5b8040..ea2ba70be15 100644 --- a/trie/zk_trie.go +++ b/trie/zk_trie.go @@ -239,15 +239,15 @@ func (t *ZkTrie) Witness() map[string]struct{} { panic("not implemented") } -func (t *ZkTrie) CountLeaves(cb func(key, value []byte), parallel bool) uint64 { +func (t *ZkTrie) CountLeaves(cb func(key, value []byte), parallel, verifyNodeHashes bool) uint64 { root, err := t.ZkTrie.Tree().Root() if err != nil { panic("CountLeaves cannot get root") } - return t.countLeaves(root, cb, 0, parallel) + return t.countLeaves(root, cb, 0, parallel, verifyNodeHashes) } -func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth int, parallel bool) uint64 { +func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth int, parallel, verifyNodeHashes bool) uint64 { if root == nil { return 0 } @@ -258,6 +258,16 @@ func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth i } if rootNode.Type == zktrie.NodeTypeLeaf_New { + if verifyNodeHashes { + calculatedNodeHash, err := rootNode.NodeHash() + if err != nil { + panic("countLeaves cannot get calculatedNodeHash") + } + if *calculatedNodeHash != *root { + panic("countLeaves node hash mismatch") + } + } + cb(append([]byte{}, rootNode.NodeKey.Bytes()...), append([]byte{}, rootNode.Data()...)) return 1 } else { @@ -266,14 +276,14 @@ func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth i leftT := t.Copy() rightT := t.Copy() go func() { - count <- leftT.countLeaves(rootNode.ChildL, cb, depth+1, parallel) + count <- leftT.countLeaves(rootNode.ChildL, cb, depth+1, parallel, verifyNodeHashes) }() go func() { - count <- rightT.countLeaves(rootNode.ChildR, cb, depth+1, parallel) + count <- rightT.countLeaves(rootNode.ChildR, cb, depth+1, parallel, verifyNodeHashes) }() return <-count + <-count } else { - return t.countLeaves(rootNode.ChildL, cb, depth+1, parallel) + t.countLeaves(rootNode.ChildR, cb, depth+1, parallel) + return t.countLeaves(rootNode.ChildL, cb, depth+1, parallel, verifyNodeHashes) + t.countLeaves(rootNode.ChildR, cb, depth+1, parallel, verifyNodeHashes) } } } From 642c35f9906cab9001ac77190e198d18e2a044b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Fri, 21 Feb 2025 17:31:11 +0300 Subject: [PATCH 04/12] make linter happy --- cmd/migration-checker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index febfba1eea2..49922c539be 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -105,7 +105,7 @@ func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leaf mptKey := crypto.Keccak256([]byte(preimageKey)) mptVal, ok := mptLeafMap[string(mptKey)] if !ok { - panic(fmt.Sprintf("%s key %s (preimage %s) not found in mpt", label, hex.EncodeToString([]byte(mptKey)), hex.EncodeToString([]byte(preimageKey)))) + panic(fmt.Sprintf("%s key %s (preimage %s) not found in mpt", label, hex.EncodeToString(mptKey), hex.EncodeToString([]byte(preimageKey)))) } leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(preimageKey))), dbs, zkValue, mptVal, paranoid) From 8a7e8ca92ec793eb14313a0f6548a0d16b02b029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Tue, 11 Mar 2025 13:33:49 +0300 Subject: [PATCH 05/12] print some more stats --- cmd/migration-checker/main.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 49922c539be..b3a16f75264 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -9,6 +9,7 @@ import ( "runtime" "sync" "sync/atomic" + "time" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" @@ -48,6 +49,20 @@ func main() { trieCheckers <- struct{}{} } + done := make(chan struct{}) + totalCheckers := len(trieCheckers) + go func() { + for { + select { + case <-done: + return + case <-time.After(time.Minute): + fmt.Println("Active checkers:", totalCheckers-len(trieCheckers)) + } + } + }() + defer close(done) + checkTrieEquality(&dbs{ zkDb: zkDb, mptDb: mptDb, @@ -68,6 +83,22 @@ func dup(s []byte) []byte { return append([]byte{}, s...) } func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte, bool), top, paranoid bool) { + done := make(chan struct{}) + start := time.Now() + if !top { + go func() { + for { + select { + case <-done: + return + case <-time.After(time.Minute): + fmt.Println("Checking trie", label, "for", time.Since(start)) + } + } + }() + } + defer close(done) + zkTrie, err := trie.NewZkTrie(zkRoot, trie.NewZktrieDatabaseFromTriedb(trie.NewDatabaseWithConfig(dbs.zkDb, &trie.Config{Preimages: true}))) panicOnError(err, label, "failed to create zk trie") mptTrie, err := trie.NewSecureNoTracer(mptRoot, trie.NewDatabaseWithConfig(dbs.mptDb, &trie.Config{Preimages: true})) From d623b3ffb9ec13020daab58b91a044db43f62e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Tue, 11 Mar 2025 14:31:31 +0300 Subject: [PATCH 06/12] allow configuring paralelism --- cmd/migration-checker/main.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index b3a16f75264..97e6e5a6431 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -20,7 +20,7 @@ import ( ) var accountsDone atomic.Uint64 -var trieCheckers = make(chan struct{}, runtime.GOMAXPROCS(0)*4) +var trieCheckers chan struct{} type dbs struct { zkDb *leveldb.Database @@ -29,11 +29,12 @@ type dbs struct { func main() { var ( - mptDbPath = flag.String("mpt-db", "", "path to the MPT node DB") - zkDbPath = flag.String("zk-db", "", "path to the ZK node DB") - mptRoot = flag.String("mpt-root", "", "root hash of the MPT node") - zkRoot = flag.String("zk-root", "", "root hash of the ZK node") - paranoid = flag.Bool("paranoid", false, "verifies all node contents against their expected hash") + mptDbPath = flag.String("mpt-db", "", "path to the MPT node DB") + zkDbPath = flag.String("zk-db", "", "path to the ZK node DB") + mptRoot = flag.String("mpt-root", "", "root hash of the MPT node") + zkRoot = flag.String("zk-root", "", "root hash of the ZK node") + paranoid = flag.Bool("paranoid", false, "verifies all node contents against their expected hash") + parallelismMultipler = flag.Int("parallelism-multiplier", 4, "multiplier for the number of parallel workers") ) flag.Parse() @@ -45,7 +46,9 @@ func main() { zkRootHash := common.HexToHash(*zkRoot) mptRootHash := common.HexToHash(*mptRoot) - for i := 0; i < runtime.GOMAXPROCS(0)*4; i++ { + numTrieCheckers := runtime.GOMAXPROCS(0) * (*parallelismMultipler) + trieCheckers = make(chan struct{}, numTrieCheckers) + for i := 0; i < numTrieCheckers; i++ { trieCheckers <- struct{}{} } @@ -68,7 +71,7 @@ func main() { mptDb: mptDb, }, zkRootHash, mptRootHash, "", checkAccountEquality, true, *paranoid) - for i := 0; i < runtime.GOMAXPROCS(0)*4; i++ { + for i := 0; i < numTrieCheckers; i++ { <-trieCheckers } } From 6ed543b28265249b6bf7635e94c21271d6cc07bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Wed, 19 Mar 2025 13:35:00 +0300 Subject: [PATCH 07/12] allow parallel storage trie loading --- cmd/migration-checker/main.go | 30 +++++++++++------------------- trie/zk_trie.go | 19 ++++++++++++------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 97e6e5a6431..b58e39782fb 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -199,11 +199,11 @@ func checkStorageEquality(label string, _ *dbs, zkStorageBytes, mptStorageBytes } } -func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { +func loadMPT(mptTrie *trie.SecureTrie, top bool) chan map[string][]byte { startKey := make([]byte, 32) workers := 1 << 5 - if !parallel { - workers = 1 + if !top { + workers = 1 << 3 } step := byte(0xFF) / byte(workers) @@ -219,9 +219,7 @@ func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { go func() { defer mptWg.Done() for trieIt.Next() { - if parallel { - mptLeafMutex.Lock() - } + mptLeafMutex.Lock() if _, ok := mptLeafMap[string(trieIt.Key)]; ok { mptLeafMutex.Unlock() @@ -230,11 +228,9 @@ func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { mptLeafMap[string(dup(trieIt.Key))] = dup(trieIt.Value) - if parallel { - mptLeafMutex.Unlock() - } + mptLeafMutex.Unlock() - if parallel && len(mptLeafMap)%10000 == 0 { + if top && len(mptLeafMap)%10000 == 0 { fmt.Println("MPT Accounts Loaded:", len(mptLeafMap)) } } @@ -249,7 +245,7 @@ func loadMPT(mptTrie *trie.SecureTrie, parallel bool) chan map[string][]byte { return respChan } -func loadZkTrie(zkTrie *trie.ZkTrie, parallel, paranoid bool) chan map[string][]byte { +func loadZkTrie(zkTrie *trie.ZkTrie, top, paranoid bool) chan map[string][]byte { zkLeafMap := make(map[string][]byte, 1000) var zkLeafMutex sync.Mutex zkDone := make(chan map[string][]byte) @@ -260,20 +256,16 @@ func loadZkTrie(zkTrie *trie.ZkTrie, parallel, paranoid bool) chan map[string][] panic(fmt.Sprintf("preimage not found zk trie %s", hex.EncodeToString(key))) } - if parallel { - zkLeafMutex.Lock() - } + zkLeafMutex.Lock() zkLeafMap[string(dup(preimageKey))] = value - if parallel { - zkLeafMutex.Unlock() - } + zkLeafMutex.Unlock() - if parallel && len(zkLeafMap)%10000 == 0 { + if top && len(zkLeafMap)%10000 == 0 { fmt.Println("ZK Accounts Loaded:", len(zkLeafMap)) } - }, parallel, paranoid) + }, top, paranoid) zkDone <- zkLeafMap }() return zkDone diff --git a/trie/zk_trie.go b/trie/zk_trie.go index ea2ba70be15..b9de8c9238a 100644 --- a/trie/zk_trie.go +++ b/trie/zk_trie.go @@ -239,15 +239,15 @@ func (t *ZkTrie) Witness() map[string]struct{} { panic("not implemented") } -func (t *ZkTrie) CountLeaves(cb func(key, value []byte), parallel, verifyNodeHashes bool) uint64 { +func (t *ZkTrie) CountLeaves(cb func(key, value []byte), top, verifyNodeHashes bool) uint64 { root, err := t.ZkTrie.Tree().Root() if err != nil { panic("CountLeaves cannot get root") } - return t.countLeaves(root, cb, 0, parallel, verifyNodeHashes) + return t.countLeaves(root, cb, 0, top, verifyNodeHashes) } -func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth int, parallel, verifyNodeHashes bool) uint64 { +func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth int, top, verifyNodeHashes bool) uint64 { if root == nil { return 0 } @@ -271,19 +271,24 @@ func (t *ZkTrie) countLeaves(root *zkt.Hash, cb func(key, value []byte), depth i cb(append([]byte{}, rootNode.NodeKey.Bytes()...), append([]byte{}, rootNode.Data()...)) return 1 } else { - if parallel && depth < 5 { + parallelismDepth := 5 + if !top { + parallelismDepth = 3 + } + + if depth < parallelismDepth { count := make(chan uint64) leftT := t.Copy() rightT := t.Copy() go func() { - count <- leftT.countLeaves(rootNode.ChildL, cb, depth+1, parallel, verifyNodeHashes) + count <- leftT.countLeaves(rootNode.ChildL, cb, depth+1, top, verifyNodeHashes) }() go func() { - count <- rightT.countLeaves(rootNode.ChildR, cb, depth+1, parallel, verifyNodeHashes) + count <- rightT.countLeaves(rootNode.ChildR, cb, depth+1, top, verifyNodeHashes) }() return <-count + <-count } else { - return t.countLeaves(rootNode.ChildL, cb, depth+1, parallel, verifyNodeHashes) + t.countLeaves(rootNode.ChildR, cb, depth+1, parallel, verifyNodeHashes) + return t.countLeaves(rootNode.ChildL, cb, depth+1, top, verifyNodeHashes) + t.countLeaves(rootNode.ChildR, cb, depth+1, top, verifyNodeHashes) } } } From b006ae5681d1fd82eaaa89423fbe0e1f578cbc59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Thu, 20 Mar 2025 12:01:40 +0300 Subject: [PATCH 08/12] move to slices --- cmd/migration-checker/main.go | 110 ++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index b58e39782fb..0397dda22ae 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -7,13 +7,13 @@ import ( "fmt" "os" "runtime" + "sort" "sync" "sync/atomic" "time" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" - "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/ethdb/leveldb" "github.com/scroll-tech/go-ethereum/rlp" "github.com/scroll-tech/go-ethereum/trie" @@ -110,39 +110,16 @@ func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leaf mptLeafCh := loadMPT(mptTrie, top) zkLeafCh := loadZkTrie(zkTrie, top, paranoid) - mptLeafMap := <-mptLeafCh - zkLeafMap := <-zkLeafCh + mptLeafs := <-mptLeafCh + zkLeafs := <-zkLeafCh - if len(mptLeafMap) != len(zkLeafMap) { - panic(fmt.Sprintf("%s MPT and ZK trie leaf count mismatch: MPT: %d, ZK: %d", label, len(mptLeafMap), len(zkLeafMap))) + if len(mptLeafs) != len(zkLeafs) { + panic(fmt.Sprintf("%s MPT and ZK trie leaf count mismatch: MPT: %d, ZK: %d", label, len(mptLeafs), len(zkLeafs))) } - for preimageKey, zkValue := range zkLeafMap { - if top { - // ZkTrie pads preimages with 0s to make them 32 bytes. - // So we might need to clear those zeroes here since we need 20 byte addresses at top level (ie state trie) - if len(preimageKey) > 20 { - for _, b := range []byte(preimageKey)[20:] { - if b != 0 { - panic(fmt.Sprintf("%s padded byte is not 0 (preimage %s)", label, hex.EncodeToString([]byte(preimageKey)))) - } - } - preimageKey = preimageKey[:20] - } - } else if len(preimageKey) != 32 { - // storage leafs should have 32 byte keys, pad them if needed - zeroes := make([]byte, 32) - copy(zeroes, []byte(preimageKey)) - preimageKey = string(zeroes) - } - - mptKey := crypto.Keccak256([]byte(preimageKey)) - mptVal, ok := mptLeafMap[string(mptKey)] - if !ok { - panic(fmt.Sprintf("%s key %s (preimage %s) not found in mpt", label, hex.EncodeToString(mptKey), hex.EncodeToString([]byte(preimageKey)))) - } - - leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(preimageKey))), dbs, zkValue, mptVal, paranoid) + for index, zkKv := range zkLeafs { + mptKv := mptLeafs[index] + leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(zkKv.key))), dbs, zkKv.value, mptKv.value, paranoid) } } @@ -199,7 +176,11 @@ func checkStorageEquality(label string, _ *dbs, zkStorageBytes, mptStorageBytes } } -func loadMPT(mptTrie *trie.SecureTrie, top bool) chan map[string][]byte { +type kv struct { + key, value []byte +} + +func loadMPT(mptTrie *trie.SecureTrie, top bool) chan []kv { startKey := make([]byte, 32) workers := 1 << 5 if !top { @@ -207,7 +188,7 @@ func loadMPT(mptTrie *trie.SecureTrie, top bool) chan map[string][]byte { } step := byte(0xFF) / byte(workers) - mptLeafMap := make(map[string][]byte, 1000) + mptLeafs := make([]kv, 0, 1000) var mptLeafMutex sync.Mutex var mptWg sync.WaitGroup @@ -215,40 +196,45 @@ func loadMPT(mptTrie *trie.SecureTrie, top bool) chan map[string][]byte { startKey[0] = byte(i) * step trieIt := trie.NewIterator(mptTrie.NodeIterator(startKey)) + stopKey := byte(i+1) * step mptWg.Add(1) go func() { defer mptWg.Done() for trieIt.Next() { - mptLeafMutex.Lock() - - if _, ok := mptLeafMap[string(trieIt.Key)]; ok { - mptLeafMutex.Unlock() + if trieIt.Key[0] >= stopKey { break } - mptLeafMap[string(dup(trieIt.Key))] = dup(trieIt.Value) + preimageKey := mptTrie.GetKey(trieIt.Key) + if len(preimageKey) == 0 { + panic(fmt.Sprintf("preimage not found mpt trie %s", hex.EncodeToString(trieIt.Key))) + } + mptLeafMutex.Lock() + mptLeafs = append(mptLeafs, kv{key: preimageKey, value: dup(trieIt.Value)}) mptLeafMutex.Unlock() - - if top && len(mptLeafMap)%10000 == 0 { - fmt.Println("MPT Accounts Loaded:", len(mptLeafMap)) + if top && len(mptLeafs)%10000 == 0 { + fmt.Println("MPT Accounts Loaded:", len(mptLeafs)) } } }() } - respChan := make(chan map[string][]byte) + respChan := make(chan []kv) go func() { mptWg.Wait() - respChan <- mptLeafMap + sort.Slice(mptLeafs, func(i, j int) bool { + return bytes.Compare(mptLeafs[i].key, mptLeafs[j].key) < 0 + }) + respChan <- mptLeafs }() return respChan } -func loadZkTrie(zkTrie *trie.ZkTrie, top, paranoid bool) chan map[string][]byte { - zkLeafMap := make(map[string][]byte, 1000) +func loadZkTrie(zkTrie *trie.ZkTrie, top, paranoid bool) chan []kv { + zkLeafs := make([]kv, 0, 1000) var zkLeafMutex sync.Mutex - zkDone := make(chan map[string][]byte) + zkDone := make(chan []kv) go func() { zkTrie.CountLeaves(func(key, value []byte) { preimageKey := zkTrie.GetKey(key) @@ -256,17 +242,37 @@ func loadZkTrie(zkTrie *trie.ZkTrie, top, paranoid bool) chan map[string][]byte panic(fmt.Sprintf("preimage not found zk trie %s", hex.EncodeToString(key))) } - zkLeafMutex.Lock() - - zkLeafMap[string(dup(preimageKey))] = value + if top { + // ZkTrie pads preimages with 0s to make them 32 bytes. + // So we might need to clear those zeroes here since we need 20 byte addresses at top level (ie state trie) + if len(preimageKey) > 20 { + for _, b := range []byte(preimageKey)[20:] { + if b != 0 { + panic(fmt.Sprintf("padded byte is not 0 (preimage %s)", hex.EncodeToString([]byte(preimageKey)))) + } + } + preimageKey = preimageKey[:20] + } + } else if len(preimageKey) != 32 { + // storage leafs should have 32 byte keys, pad them if needed + zeroes := make([]byte, 32) + copy(zeroes, []byte(preimageKey)) + preimageKey = zeroes + } + zkLeafMutex.Lock() + zkLeafs = append(zkLeafs, kv{key: preimageKey, value: value}) zkLeafMutex.Unlock() - if top && len(zkLeafMap)%10000 == 0 { - fmt.Println("ZK Accounts Loaded:", len(zkLeafMap)) + if top && len(zkLeafs)%10000 == 0 { + fmt.Println("ZK Accounts Loaded:", len(zkLeafs)) } }, top, paranoid) - zkDone <- zkLeafMap + + sort.Slice(zkLeafs, func(i, j int) bool { + return bytes.Compare(zkLeafs[i].key, zkLeafs[j].key) < 0 + }) + zkDone <- zkLeafs }() return zkDone } From 08a049bd801ef786e195917443ee97fadb670c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Thu, 20 Mar 2025 12:40:54 +0300 Subject: [PATCH 09/12] fix mpt loading --- cmd/migration-checker/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 0397dda22ae..cd80976bf25 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -186,7 +186,7 @@ func loadMPT(mptTrie *trie.SecureTrie, top bool) chan []kv { if !top { workers = 1 << 3 } - step := byte(0xFF) / byte(workers) + step := byte(256 / workers) mptLeafs := make([]kv, 0, 1000) var mptLeafMutex sync.Mutex @@ -196,12 +196,12 @@ func loadMPT(mptTrie *trie.SecureTrie, top bool) chan []kv { startKey[0] = byte(i) * step trieIt := trie.NewIterator(mptTrie.NodeIterator(startKey)) - stopKey := byte(i+1) * step + stopKey := (i + 1) * int(step) mptWg.Add(1) go func() { defer mptWg.Done() for trieIt.Next() { - if trieIt.Key[0] >= stopKey { + if int(trieIt.Key[0]) >= stopKey { break } From c933b5c81855da85479d77a089f8676fd2346df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Thu, 20 Mar 2025 12:56:57 +0300 Subject: [PATCH 10/12] progress --- cmd/migration-checker/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index cd80976bf25..0811bc0018d 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -20,6 +20,7 @@ import ( ) var accountsDone atomic.Uint64 +var totalAccounts int var trieCheckers chan struct{} type dbs struct { @@ -115,6 +116,8 @@ func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leaf if len(mptLeafs) != len(zkLeafs) { panic(fmt.Sprintf("%s MPT and ZK trie leaf count mismatch: MPT: %d, ZK: %d", label, len(mptLeafs), len(zkLeafs))) + } else if top { + totalAccounts = len(mptLeafs) } for index, zkKv := range zkLeafs { @@ -157,12 +160,12 @@ func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountByte checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false, paranoid) accountsDone.Add(1) - fmt.Println("Accounts done:", accountsDone.Load()) + fmt.Println("Accounts done:", accountsDone.Load(), "/", totalAccounts) trieCheckers <- struct{}{} }() } else { accountsDone.Add(1) - fmt.Println("Accounts done:", accountsDone.Load()) + fmt.Println("Accounts done:", accountsDone.Load(), "/", totalAccounts) } } From 8e0b29cebb69db9adaf5e534a0cb503e0eed5f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Thu, 20 Mar 2025 13:03:20 +0300 Subject: [PATCH 11/12] add start from --- cmd/migration-checker/main.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 0811bc0018d..4c00fbb14c4 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -36,6 +36,7 @@ func main() { zkRoot = flag.String("zk-root", "", "root hash of the ZK node") paranoid = flag.Bool("paranoid", false, "verifies all node contents against their expected hash") parallelismMultipler = flag.Int("parallelism-multiplier", 4, "multiplier for the number of parallel workers") + startFrom = flag.Int("start-form", 0, "start checking from account at the given index") ) flag.Parse() @@ -67,10 +68,16 @@ func main() { }() defer close(done) + startFromSafe := *startFrom - len(trieCheckers) + if startFromSafe < 0 { + startFromSafe = 0 + } + accountsDone.Add(uint64(startFromSafe)) + checkTrieEquality(&dbs{ zkDb: zkDb, mptDb: mptDb, - }, zkRootHash, mptRootHash, "", checkAccountEquality, true, *paranoid) + }, zkRootHash, mptRootHash, "", checkAccountEquality, true, *paranoid, startFromSafe) for i := 0; i < numTrieCheckers; i++ { <-trieCheckers @@ -86,7 +93,7 @@ func panicOnError(err error, label, msg string) { func dup(s []byte) []byte { return append([]byte{}, s...) } -func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte, bool), top, paranoid bool) { +func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leafChecker func(string, *dbs, []byte, []byte, bool), top, paranoid bool, startFrom int) { done := make(chan struct{}) start := time.Now() if !top { @@ -120,8 +127,9 @@ func checkTrieEquality(dbs *dbs, zkRoot, mptRoot common.Hash, label string, leaf totalAccounts = len(mptLeafs) } - for index, zkKv := range zkLeafs { - mptKv := mptLeafs[index] + for i := startFrom; i < len(zkLeafs); i++ { + zkKv := zkLeafs[i] + mptKv := mptLeafs[i] leafChecker(fmt.Sprintf("%s key: %s", label, hex.EncodeToString([]byte(zkKv.key))), dbs, zkKv.value, mptKv.value, paranoid) } } @@ -158,7 +166,7 @@ func checkAccountEquality(label string, dbs *dbs, zkAccountBytes, mptAccountByte } }() - checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false, paranoid) + checkTrieEquality(dbs, zkRoot, mptRoot, label, checkStorageEquality, false, paranoid, 0) accountsDone.Add(1) fmt.Println("Accounts done:", accountsDone.Load(), "/", totalAccounts) trieCheckers <- struct{}{} From 372533b352c745eaa1dc024c4865e26c91bbcdc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Fri, 21 Mar 2025 10:06:34 +0300 Subject: [PATCH 12/12] ++ --- cmd/migration-checker/main.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/cmd/migration-checker/main.go b/cmd/migration-checker/main.go index 4c00fbb14c4..5bf32dce28b 100644 --- a/cmd/migration-checker/main.go +++ b/cmd/migration-checker/main.go @@ -36,7 +36,7 @@ func main() { zkRoot = flag.String("zk-root", "", "root hash of the ZK node") paranoid = flag.Bool("paranoid", false, "verifies all node contents against their expected hash") parallelismMultipler = flag.Int("parallelism-multiplier", 4, "multiplier for the number of parallel workers") - startFrom = flag.Int("start-form", 0, "start checking from account at the given index") + startFrom = flag.Int("start-from", 0, "start checking from account at the given index") ) flag.Parse() @@ -54,20 +54,6 @@ func main() { trieCheckers <- struct{}{} } - done := make(chan struct{}) - totalCheckers := len(trieCheckers) - go func() { - for { - select { - case <-done: - return - case <-time.After(time.Minute): - fmt.Println("Active checkers:", totalCheckers-len(trieCheckers)) - } - } - }() - defer close(done) - startFromSafe := *startFrom - len(trieCheckers) if startFromSafe < 0 { startFromSafe = 0