Track Soroban in-memory state size and store it in p23. (#4809)

# Description

This adds the state size tracking mechanism to InMemorySorobanState. The
size includes the size of the parsed Wasm modules and thus has to be
recomputed on the protocol and relevant setting upgrades. We'll start
tracking the state size as soon as we're in Soroban protocol, but will
only use it in p23. Because of that we can use p23 host to compute the
memory cost for contract code entries eagerly.

This does a few non-trivial things on upgrades:

- When we perform a memory cost settings upgrade (even in p22) we
recompute the contract code in-memory size. If we are in p23+, we also
fill the state size window with the current in-memory state size.
- When we upgrade to p23, we immediately fill the state size window with
the current in-memory state size, but don't recompute the contract code
size (it's already up to date)
- When we upgrade to p24+, we recompute the contract code size and fill
the state window with the current in-memory state size

All these cases are covered in the respective new tests.

There is also a bunch of test-related TODOs here that can be addressed
later - they shouldn't influence the change coverage significantly.

# Checklist
- [ ] Reviewed the
[contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes)
document
- [ ] Rebased on top of master (no merge commits)
- [ ] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio
extension)
- [ ] Compiles
- [ ] Ran all tests
- [ ] If change impacts performance, include supporting evidence per the
[performance
document](https://github.com/stellar/stellar-core/blob/master/performance-eval/performance-eval.md)

Author: dmkozh

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -1411,7 +1411,9 @@
     <ClCompile Include="..\..\src\transactions\ParallelApplyUtils.cpp">
       <Filter>transactions</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\ledger\InMemorySorobanState.cpp" />
+    <ClCompile Include="..\..\src\ledger\InMemorySorobanState.cpp">
+      <Filter>ledger</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\lib\util\cpptoml.h">
@@ -2504,6 +2506,9 @@
       <Filter>transactions</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\ledger\InMemorySorobanState.h" />
+    <ClInclude Include="..\..\src\ledger\InMemorySorobanState.h">
+      <Filter>ledger</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\AUTHORS" />

src/bucket/HotArchiveBucketIndex.h

@@ -94,7 +94,7 @@ class HotArchiveBucketIndex : public NonMovableOrCopyable
     uint32_t
     getPageSize() const
     {
-        return mDiskIndex.getPageSize();
+        return static_cast<uint32_t>(mDiskIndex.getPageSize());
     }
 
     IterT
@@ -118,4 +118,4 @@ class HotArchiveBucketIndex : public NonMovableOrCopyable
     bool operator==(HotArchiveBucketIndex const& in) const;
 #endif
 };
-}
+}

src/bucket/test/BucketIndexTests.cpp

@@ -1159,7 +1159,8 @@ TEST_CASE("soroban cache population", "[soroban][bucketindex]")
 
         // Now wipe cache and repopulate from scratch to test initialization on
         // a non-empty bucketlist.
-        lm.rebuildInMemorySorobanStateForTesting();
+        lm.rebuildInMemorySorobanStateForTesting(
+            lm.getLastClosedLedgerHeader().header.ledgerVersion);
         testCache();
     };
 

src/bucket/test/BucketListTests.cpp

@@ -878,10 +878,13 @@ TEST_CASE("BucketList check bucket sizes", "[bucket][bucketlist][count]")
     }
 }
 
-TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
+TEST_CASE_VERSIONS("network config snapshots Soroban state size", "[soroban]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY));
+    // TODO(https://github.com/stellar/stellar-core/issues/4816): We should
+    // be using the default DB mode here, and also update the window size to
+    // make sure the upgrades work correctly.
+    Config cfg(getTestConfig(0, Config::TestDbMode::TESTDB_IN_MEMORY));
     cfg.USE_CONFIG_FOR_GENESIS = true;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);
@@ -891,6 +894,11 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
         auto networkConfig = [&]() {
             return app->getLedgerManager().getLastClosedSorobanNetworkConfig();
         };
+        // Take snapshots more frequently for faster testing.
+        modifySorobanNetworkConfig(*app, [](SorobanNetworkConfig& cfg) {
+            cfg.mStateArchivalSettings.liveSorobanStateSizeWindowSamplePeriod =
+                7;
+        });
 
         uint32_t windowSize = networkConfig()
                                   .stateArchivalSettings()
@@ -901,19 +909,28 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
             correctWindow.push_back(0);
         }
 
-        auto check = [&]() {
+        auto check = [&](bool init = false) {
             // Check in-memory average from BucketManager
             uint64_t sum = 0;
-            for (auto e : correctWindow)
+            for (int i = 0; i < correctWindow.size(); ++i)
             {
-                sum += e;
+                // Ensure that we're actually increasing the state size.
+                if (i < correctWindow.size() - 1 && correctWindow[i] != 0)
+                {
+                    REQUIRE(correctWindow[i] < correctWindow[i + 1]);
+                }
+                sum += correctWindow[i];
             }
 
             uint64_t correctAverage = sum / correctWindow.size();
+            if (!init)
+            {
+                REQUIRE(correctAverage > 0);
+            }
 
             LedgerTxn ltx(app->getLedgerTxnRoot());
-            REQUIRE(networkConfig().getAverageBucketListSize() ==
-                    correctAverage);
+            auto const& networkCfg = networkConfig();
+            REQUIRE(networkCfg.getAverageSorobanStateSize() == correctAverage);
 
             // Check on-disk sliding window
             LedgerKey key(CONFIG_SETTING);
@@ -930,13 +947,7 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
         };
 
         // Check initial conditions
-        check();
-
-        // Take snapshots more frequently for faster testing
-        modifySorobanNetworkConfig(*app, [](SorobanNetworkConfig& cfg) {
-            cfg.mStateArchivalSettings.liveSorobanStateSizeWindowSamplePeriod =
-                64;
-        });
+        check(true);
 
         // Generate enough ledgers to fill sliding window
         auto ledgersToGenerate =
@@ -945,6 +956,7 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
                                    .liveSorobanStateSizeWindowSamplePeriod;
         auto lclSeq = lm.getLastClosedLedgerHeader().header.ledgerSeq;
         UnorderedSet<LedgerKey> generatedKeys;
+        int const entryCount = 10;
         for (uint32_t ledger = lclSeq; ledger < ledgersToGenerate; ++ledger)
         {
             // Note: BucketList size in the sliding window is snapshotted before
@@ -957,16 +969,33 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
                 0)
             {
                 correctWindow.pop_front();
-                correctWindow.push_back(
-                    app->getBucketManager().getLiveBucketList().getSize());
+                uint64_t sizeSnapshot = 0;
+                if (protocolVersionStartsFrom(app->getLedgerManager()
+                                                  .getLastClosedLedgerHeader()
+                                                  .header.ledgerVersion,
+                                              ProtocolVersion::V_23))
+                {
+                    sizeSnapshot =
+                        app->getLedgerManager().getSorobanInMemoryStateSize();
+                }
+                else
+                {
+                    sizeSnapshot =
+                        app->getBucketManager().getLiveBucketList().getSize();
+                }
+                correctWindow.push_back(sizeSnapshot);
             }
 
-            // Exclude soroban types to avoid TTL invariants
-            lm.setNextLedgerEntryBatchForBucketTesting(
-                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
-                    {CONFIG_SETTING, TTL, CONTRACT_DATA, CONTRACT_CODE}, 10,
-                    generatedKeys),
-                {}, {});
+            auto entries =
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithTypes(
+                    {CONTRACT_DATA, CONTRACT_CODE}, entryCount, generatedKeys);
+            for (int i = 0; i < entryCount; ++i)
+            {
+                entries.push_back(
+                    getTTLEntryForTTLKey(getTTLKey(entries[i]), 100'000));
+            }
+
+            lm.setNextLedgerEntryBatchForBucketTesting(entries, {}, {});
             closeLedger(*app);
             if ((ledger + 1) % networkConfig()
                                    .stateArchivalSettings()

src/bucket/test/BucketTestUtils.cpp

@@ -275,7 +275,10 @@ LedgerManagerForBucketTests::sealLedgerTxnAndTransferEntriesToBucketList(
             }
             mApp.getLedgerManager()
                 .getMutableSorobanNetworkConfigForApply()
-                .maybeSnapshotBucketListSize(lh.ledgerSeq, ltx, mApp);
+                .maybeSnapshotSorobanStateSize(
+                    lh.ledgerSeq,
+                    mApp.getLedgerManager().getSorobanInMemoryStateSize(), ltx,
+                    mApp);
         }
 
         ltx.getAllEntries(init, live, dead);

src/herder/Upgrades.cpp

@@ -1182,16 +1182,16 @@ void
 Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
                               uint32_t newVersion)
 {
-    auto header = ltx.loadHeader();
-    uint32_t prevVersion = header.current().ledgerVersion;
+    uint32_t prevVersion = ltx.loadHeader().current().ledgerVersion;
 
-    header.current().ledgerVersion = newVersion;
+    ltx.loadHeader().current().ledgerVersion = newVersion;
     if (needUpgradeToVersion(ProtocolVersion::V_10, prevVersion, newVersion))
     {
+        auto header = ltx.loadHeader();
         prepareLiabilities(ltx, header);
     }
-    if (protocolVersionEquals(header.current().ledgerVersion,
-                              ProtocolVersion::V_16) &&
+
+    if (protocolVersionEquals(newVersion, ProtocolVersion::V_16) &&
         protocolVersionEquals(prevVersion, ProtocolVersion::V_15))
     {
         upgradeFromProtocol15To16(ltx);
@@ -1212,6 +1212,7 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
         }
 #endif
     }
+
     if (needUpgradeToVersion(ProtocolVersion::V_21, prevVersion, newVersion))
     {
         SorobanNetworkConfig::createCostTypesForV21(ltx, app);
@@ -1224,6 +1225,15 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx,
     {
         SorobanNetworkConfig::createAndUpdateLedgerEntriesForV23(ltx, app);
     }
+
+    // Starting from protocol 23 we need to fully override the Soroban in-memory
+    // state size on upgrade, as before protocol 23 bucket list size has bene
+    // used.
+    if (protocolVersionStartsFrom(newVersion, ProtocolVersion::V_23))
+    {
+        app.getLedgerManager().handleUpgradeAffectingSorobanInMemoryStateSize(
+            ltx);
+    }
 }
 
 void
@@ -1391,32 +1401,42 @@ void
 ConfigUpgradeSetFrame::applyTo(AbstractLedgerTxn& ltx, Application& app) const
 {
     bool writeLiveSorobanStateSizeWindow = false;
+    bool hasMemorySettingsUpgrade = false;
     for (auto const& updatedEntry : mConfigUpgradeSet.updatedEntry)
     {
         LedgerKey key(LedgerEntryType::CONFIG_SETTING);
         auto const id = updatedEntry.configSettingID();
         key.configSetting().configSettingID = id;
         auto& currentEntry = ltx.load(key).current().data.configSetting();
-        if (currentEntry.configSettingID() ==
-                ConfigSettingID::CONFIG_SETTING_STATE_ARCHIVAL &&
+        if (id == ConfigSettingID::CONFIG_SETTING_STATE_ARCHIVAL &&
             currentEntry.stateArchivalSettings()
                     .liveSorobanStateSizeWindowSampleSize !=
                 updatedEntry.stateArchivalSettings()
                     .liveSorobanStateSizeWindowSampleSize)
         {
             writeLiveSorobanStateSizeWindow = true;
         }
+        if (id ==
+            ConfigSettingID::CONFIG_SETTING_CONTRACT_COST_PARAMS_MEMORY_BYTES)
+        {
+            hasMemorySettingsUpgrade = true;
+        }
         currentEntry = updatedEntry;
     }
-
+    // If there was an upgrade for the Soroban state size window, we need to
+    // truncate or increase the size snapshot window respectively.
     if (writeLiveSorobanStateSizeWindow)
     {
         SorobanNetworkConfig networkConfig;
-        networkConfig.loadFromLedger(
-            ltx, app.getConfig().CURRENT_LEDGER_PROTOCOL_VERSION,
-            mLedgerVersion);
-
-        networkConfig.maybeUpdateBucketListWindowSize(ltx);
+        networkConfig.loadFromLedger(ltx);
+        networkConfig.maybeUpdateSorobanStateSizeWindowSize(ltx);
+    }
+    // If there was an upgrade for the memory cost settings, we need to
+    // recompute the current state size and override the old sizes.
+    if (hasMemorySettingsUpgrade)
+    {
+        app.getLedgerManager().handleUpgradeAffectingSorobanInMemoryStateSize(
+            ltx);
     }
 }