Skip to content

Commit fdd044a

Browse files
aopoltorzhickyaopoltorzhicky
committed
Fix: mempool deadlock
1 parent 393647c commit fdd044a

File tree

7 files changed

+57
-128
lines changed

7 files changed

+57
-128
lines changed

‎.vscode/launch.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"version": "0.2.0",
66
"configurations": [
77
{
8-
"name": "Launch Indexer",
8+
"name": "Launch Mempool",
99
"type": "go",
1010
"request": "launch",
1111
"mode": "debug",

‎cmd/mempool/handlers.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ import (
1818
)
1919

2020
func (indexer *Indexer) handleBlock(block tzkt.BlockMessage) error {
21+
if err := indexer.processOldOperations(); err != nil {
22+
return err
23+
}
24+
2125
switch block.Type {
2226
case events.MessageTypeState:
2327
if indexer.state.Level < block.Level {
@@ -48,6 +52,23 @@ func (indexer *Indexer) handleBlock(block tzkt.BlockMessage) error {
4852
return indexer.branches.Add(block)
4953
}
5054

55+
func (indexer *Indexer) processOldOperations() error {
56+
return indexer.db.Transaction(func(tx *gorm.DB) error {
57+
if err := models.DeleteOldOperations(tx, indexer.keepInChain, models.StatusInChain, indexer.filters.Kinds...); err != nil {
58+
return errors.Wrap(err, "DeleteOldOperations in_chain")
59+
}
60+
if err := models.DeleteOldOperations(tx, indexer.keepOperations, "", indexer.filters.Kinds...); err != nil {
61+
return errors.Wrap(err, "DeleteOldOperations")
62+
}
63+
if indexer.hasManager {
64+
if err := models.DeleteOldGasStats(tx, indexer.gasStatsLifetime); err != nil {
65+
return errors.Wrap(err, "DeleteOldGasStats")
66+
}
67+
}
68+
return nil
69+
})
70+
}
71+
5172
func (indexer *Indexer) handleInChain(operations tzkt.OperationMessage) error {
5273
return indexer.db.Transaction(func(tx *gorm.DB) error {
5374
operations.Hash.Range(func(_, operation interface{}) bool {

‎cmd/mempool/indexer.go‎

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,23 @@ import (
2121

2222
// Indexer -
2323
type Indexer struct {
24-
db *gorm.DB
25-
tzkt *tzkt.TzKT
26-
mempool *receiver.Receiver
27-
manager *Manager
28-
branches *BlockQueue
29-
cache *ccache.Cache
30-
delegates *CachedDelegates
31-
state state.State
32-
filters config.Filters
33-
network string
34-
indexName string
35-
chainID string
36-
stop chan struct{}
37-
threadsCount int
38-
hasManager bool
24+
db *gorm.DB
25+
tzkt *tzkt.TzKT
26+
mempool *receiver.Receiver
27+
branches *BlockQueue
28+
cache *ccache.Cache
29+
delegates *CachedDelegates
30+
state state.State
31+
filters config.Filters
32+
network string
33+
indexName string
34+
chainID string
35+
keepInChain uint64
36+
keepOperations uint64
37+
gasStatsLifetime uint64
38+
stop chan struct{}
39+
threadsCount int
40+
hasManager bool
3941

4042
wg sync.WaitGroup
4143
}
@@ -80,15 +82,18 @@ func NewIndexer(network string, indexerCfg config.Indexer, database generalConfi
8082
}
8183

8284
indexer := &Indexer{
83-
db: db,
84-
network: network,
85-
chainID: head.ChainID,
86-
indexName: models.MempoolIndexName(network),
87-
filters: indexerCfg.Filters,
88-
tzkt: tzkt.NewTzKT(indexerCfg.DataSource.Tzkt, indexerCfg.Filters.Accounts, indexerCfg.Filters.Kinds),
89-
mempool: memInd,
90-
cache: ccache.New(ccache.Configure().MaxSize(2 ^ 13)),
91-
threadsCount: 1,
85+
db: db,
86+
network: network,
87+
chainID: head.ChainID,
88+
indexName: models.MempoolIndexName(network),
89+
filters: indexerCfg.Filters,
90+
tzkt: tzkt.NewTzKT(indexerCfg.DataSource.Tzkt, indexerCfg.Filters.Accounts, indexerCfg.Filters.Kinds),
91+
mempool: memInd,
92+
cache: ccache.New(ccache.Configure().MaxSize(2 ^ 13)),
93+
threadsCount: 1,
94+
keepInChain: uint64(constants.TimeBetweenBlocks[0]) * settings.KeepInChainBlocks,
95+
keepOperations: uint64(constants.TimeBetweenBlocks[0]) * settings.ExpiredAfter,
96+
gasStatsLifetime: settings.GasStatsLifetime,
9297
}
9398

9499
indexer.state = state.State{
@@ -103,7 +108,6 @@ func NewIndexer(network string, indexerCfg config.Indexer, database generalConfi
103108
break
104109
}
105110
}
106-
indexer.manager = NewManager(db, settings, uint64(constants.TimeBetweenBlocks[0]), indexer.hasManager, indexerCfg.Filters.Kinds...)
107111
indexer.branches = newBlockQueue(expiredAfter, indexer.onPopBlockQueue, indexer.onRollbackBlockQueue)
108112

109113
for _, kind := range indexer.filters.Kinds {
@@ -146,8 +150,7 @@ func (indexer *Indexer) Start() error {
146150
return err
147151
}
148152

149-
go indexer.mempool.Start()
150-
indexer.manager.Start()
153+
indexer.mempool.Start()
151154

152155
return nil
153156
}
@@ -184,10 +187,6 @@ func (indexer *Indexer) Close() error {
184187
}
185188
indexer.wg.Wait()
186189

187-
if err := indexer.manager.Close(); err != nil {
188-
return err
189-
}
190-
191190
if err := indexer.tzkt.Close(); err != nil {
192191
return err
193192
}

‎cmd/mempool/manager.go‎

Lines changed: 0 additions & 93 deletions
This file was deleted.

‎cmd/mempool/tzkt/tzkt.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,8 @@ func (tzkt *TzKT) getTableData(table *tableState, indexerState, headLevel uint64
502502
return getOperations(table, filters, tzkt.api.GetReveals)
503503
case api.KindTransaction:
504504
return getOperations(table, filters, tzkt.api.GetTransactions)
505+
case api.KindRegisterGlobalConstant:
506+
return getOperations(table, filters, tzkt.api.GetRegisterConstants)
505507
default:
506508
return errors.Wrap(ErrUnknownOperationKind, table.Table)
507509
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.15
44

55
require (
66
github.com/btcsuite/btcutil v1.0.2
7-
github.com/dipdup-net/go-lib v0.1.31
7+
github.com/dipdup-net/go-lib v0.1.32
88
github.com/json-iterator/go v1.1.12 // indirect
99
github.com/karlseguin/ccache v2.0.3+incompatible
1010
github.com/karlseguin/expect v1.0.8 // indirect

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
6161
github.com/denisenkom/go-mssqldb v0.10.0 h1:QykgLZBorFE95+gO3u9esLd0BmbvpWp0/waNNZfHBM8=
6262
github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
6363
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
64-
github.com/dipdup-net/go-lib v0.1.31 h1:+m2W74TfC5SYIN13obXvm9CUx6aQ4ZDl3QBheHrgXU4=
65-
github.com/dipdup-net/go-lib v0.1.31/go.mod h1:i3p/qLTxQ43aiHxlg30FTLeysryjPlk5sT8fY3PxgUY=
64+
github.com/dipdup-net/go-lib v0.1.32 h1:BQZK0sECtpL8wz7xiuunrNQaB2OEzJ/jA11O/QC7UmE=
65+
github.com/dipdup-net/go-lib v0.1.32/go.mod h1:i3p/qLTxQ43aiHxlg30FTLeysryjPlk5sT8fY3PxgUY=
6666
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
6767
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
6868
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=

0 commit comments

Comments
 (0)