Contract one time oracle query

doc/contracts_proposals.md

@@ -120,7 +120,7 @@ struct QUTIL
         // Your code should set the state variable that the proposal is about to the accepted value.
         // This can be done as in this example taken from QUTIL:
 
-        switch (input.proposal.variableOptions.variable)
+        switch (input.proposal.data.variableOptions.variable)
         {
         case 0:
             state.smt1InvocationFee = input.acceptedValue;
@@ -348,7 +348,7 @@ PRIVATE_PROCEDURE_WITH_LOCALS(FinalizeShareholderStateVarProposals)
             if (locals.p.acceptedOption <= 0)
                 continue;
 
-            locals.p.acceptedValue = locals.p.proposal.variableOptions.value;
+            locals.p.acceptedValue = locals.p.proposal.data.variableOptions.value;
 
             CALL(FinalizeShareholderProposalSetStateVar, locals.p, output);
         }
@@ -376,8 +376,8 @@ PUBLIC_PROCEDURE(SetShareholderProposal)
 {
     // check proposal input and fees
     if (qpi.invocationReward() < setProposalFeeVarOrValue || (input.epoch
-        && (input.type != ProposalTypes::VariableYesNo || input.variableOptions.variable >= numFeeStateVariables
-            || input.variableOptions.value < 0)))
+        && (input.type != ProposalTypes::VariableYesNo || input.data.variableOptions.variable >= numFeeStateVariables
+            || input.data.variableOptions.value < 0)))
     {
         // error -> reimburse invocation reward
         qpi.transfer(qpi.invocator(), qpi.invocationReward());

src/Qubic.vcxproj

@@ -24,8 +24,10 @@
     <ClInclude Include="common_buffers.h" />
     <ClInclude Include="contracts\ComputorControlledFund.h" />
     <ClInclude Include="contracts\Qdraw.h" />
+    <ClInclude Include="contracts\qRWA.h" />
     <ClInclude Include="contracts\Qswap.h" />
     <ClInclude Include="contracts\RandomLottery.h" />
+    <ClInclude Include="contracts\QDuel.h" />
     <ClInclude Include="contracts\SupplyWatcher.h" />
     <ClInclude Include="contracts\EmptyTemplate.h" />
     <ClInclude Include="contracts\GeneralQuorumProposal.h" />
@@ -44,6 +46,8 @@
     <ClInclude Include="contracts\QRaffle.h" />
     <ClInclude Include="contracts\QBond.h" />
     <ClInclude Include="contracts\QIP.h" />
+    <ClInclude Include="contracts\QReservePool.h" />
+    <ClInclude Include="contracts\QThirtyFour.h" />
     <ClInclude Include="contracts\TestExampleA.h" />
     <ClInclude Include="contracts\TestExampleB.h" />
     <ClInclude Include="contracts\TestExampleC.h" />
@@ -70,6 +74,10 @@
     <ClInclude Include="logging\logging.h" />
     <ClInclude Include="logging\net_msg_impl.h" />
     <ClInclude Include="mining\mining.h" />
+    <ClInclude Include="mining\score_addition.h" />
+    <ClInclude Include="mining\score_common.h" />
+    <ClInclude Include="mining\score_engine.h" />
+    <ClInclude Include="mining\score_hyperidentity.h" />
     <ClInclude Include="network_core\peers.h" />
     <ClInclude Include="network_core\tcp4.h" />
     <ClInclude Include="network_messages\all.h" />
@@ -96,6 +104,8 @@
     <ClInclude Include="oracle_core\oracle_engine.h" />
     <ClInclude Include="oracle_core\oracle_interfaces_def.h" />
     <ClInclude Include="oracle_core\oracle_transactions.h" />
+    <ClInclude Include="oracle_core\snapshot_files.h" />
+    <ClInclude Include="oracle_interfaces\Mock.h" />
     <ClInclude Include="oracle_interfaces\Price.h" />
     <ClInclude Include="platform\assert.h" />
     <ClInclude Include="platform\concurrency.h" />

src/Qubic.vcxproj.filters

@@ -296,6 +296,15 @@
     <ClInclude Include="contracts\RandomLottery.h">
       <Filter>contracts</Filter>
     </ClInclude>
+    <ClInclude Include="contracts\QReservePool.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
+    <ClInclude Include="contracts\QThirtyFour.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
+    <ClInclude Include="contracts\QDuel.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
     <ClInclude Include="contract_core\pre_qpi_def.h">
       <Filter>contract_core</Filter>
     </ClInclude>
@@ -332,6 +341,30 @@
     <ClInclude Include="contract_core\execution_time_accumulator.h">
       <Filter>contract_core</Filter>
     </ClInclude>
+    <ClInclude Include="oracle_interfaces\Mock.h">
+      <Filter>oracle_interfaces</Filter>
+    </ClInclude>
+    <ClInclude Include="mining\score_common.h">
+      <Filter>mining</Filter>
+    </ClInclude>
+    <ClInclude Include="mining\score_engine.h">
+      <Filter>mining</Filter>
+    </ClInclude>
+    <ClInclude Include="mining\score_hyperidentity.h">
+      <Filter>mining</Filter>
+    </ClInclude>
+    <ClInclude Include="mining\score_addition.h">
+      <Filter>mining</Filter>
+    </ClInclude>
+    <ClInclude Include="contracts\qRWA.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
+    <ClInclude Include="contracts\SupplyWatcher_old.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\snapshot_files.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Filter Include="platform">

src/assets/assets.h

@@ -793,8 +793,8 @@ static void assetsEndEpoch()
     ACQUIRE(universeLock);
 
     // rebuild asset hash map, getting rid of all elements with zero shares
-    AssetRecord* reorgAssets = (AssetRecord*)reorgBuffer;
-    setMem(reorgAssets, ASSETS_CAPACITY * sizeof(AssetRecord), 0);
+    AssetRecord* reorgAssets = (AssetRecord*)commonBuffers.acquireBuffer(universeSizeInBytes);
+    setMem(reorgAssets, universeSizeInBytes, 0);
     for (unsigned int i = 0; i < ASSETS_CAPACITY; i++)
     {
         if (assets[i].varStruct.possession.type == POSSESSION
@@ -874,6 +874,7 @@ static void assetsEndEpoch()
         }
     }
     copyMem(assets, reorgAssets, ASSETS_CAPACITY * sizeof(AssetRecord));
+    commonBuffers.releaseBuffer(reorgAssets);
 
     setMem(assetChangeFlags, ASSETS_CAPACITY / 8, 0xFF);
 

src/common_buffers.h

@@ -3,45 +3,200 @@
 #include "platform/global_var.h"
 #include "platform/memory_util.h"
 #include "platform/assert.h"
+#include "platform/concurrency.h"
 
 #include "network_messages/entity.h"
 #include "network_messages/assets.h"
+#include "contract_core/pre_qpi_def.h"
+#include "contracts/math_lib.h"
 
 
 constexpr unsigned long long spectrumSizeInBytes = SPECTRUM_CAPACITY * sizeof(EntityRecord);
 constexpr unsigned long long universeSizeInBytes = ASSETS_CAPACITY * sizeof(AssetRecord);
-// TODO: check that max contract state size does not exceed size of spectrum or universe
-constexpr unsigned long long reorgBufferSize = (spectrumSizeInBytes >= universeSizeInBytes) ? spectrumSizeInBytes : universeSizeInBytes;
+constexpr unsigned long long defaultCommonBuffersSize = math_lib::max(MAX_CONTRACT_STATE_SIZE, math_lib::max(spectrumSizeInBytes, universeSizeInBytes));
 
-// Buffer used for reorganizing spectrum and universe hash maps, currently also used as scratchpad buffer for contracts
+// Buffer(s) used for:
+// - reorganizing spectrum and universe hash maps (tick processor)
+// - scratchpad buffer used internally in QPI::Collection, QPI::HashMap, QPI::HashSet,
+//   QPI::ProposalAndVotingByShareholders
+//   (often used in contract processor which does not run concurrently with tick processor, but now also used outside
+//   of contracts, e.g. pendingTxsPool.add() running in request processor may trigger Collection::_rebuild() which
+//   uses scratchpad)
+// - building oracle transactions in processTick() in tick processor
+// - calculateStableComputorIndex() in tick processor
+// - saving and loading of logging state
+// - DustBurnLogger used in increaseEnergy() in tick / contract processor
 // Must be large enough to fit any contract, full spectrum, and full universe!
-// No locking! Can be used in tick processor or contract processor.
-// May be only used in main processor if tick/contract processor is paused or not started yet.
-GLOBAL_VAR_DECL void* reorgBuffer GLOBAL_VAR_INIT(nullptr);
+class CommonBuffers
+{
+public:
+    // Allocate common buffers. With count > 1, multiple buffers may be used concurrently. The maximum buffer size
+    // that can be acquired is given by size.
+    bool init(unsigned int count, unsigned long long size = defaultCommonBuffersSize)
+    {
+        if (!count || !size)
+            return false;
+
+        // soft limit, just to detect mistakes in usage like init(sizeof(Object))
+        ASSERT(count < 16);
+
+        // memory layout of buffer: sub buffer pointers | sub buffer locks | sub buffer 1 | sub buffer 2 | ...
+        unsigned char* buffer = nullptr;
+        const unsigned long long ptrSize = count * sizeof(unsigned char*);
+        const unsigned long long lockSize = (count + 7) / 8;
+        const unsigned long long bufSize = count * size;
+
+        if (!allocPoolWithErrorLog(L"commonBuffers", ptrSize + lockSize + bufSize, (void**)&buffer, __LINE__))
+        {
+            return false;
+        }
 
-static bool initCommonBuffers()
-{
-    if (!allocPoolWithErrorLog(L"reorgBuffer", reorgBufferSize, (void**)&reorgBuffer, __LINE__))
+        bufferCount = count;
+        subBufferSize = size;
+        subBufferPtr = (unsigned char**)buffer;
+        subBufferLock = (volatile char*)(buffer + ptrSize);
+        unsigned char* subBuf = buffer + ptrSize + lockSize;
+        for (unsigned int i = 0; i < count; ++i)
+        {
+            subBufferPtr[i] = subBuf;
+            subBuf += size;
+        }
+
+        return true;
+    }
+
+    // Free common buffers.
+    void deinit()
     {
-        return false;
+        if (subBufferPtr)
+        {
+            freePool(subBufferPtr);
+            subBufferPtr = nullptr;
+            subBufferLock = nullptr;
+            subBufferSize = 0;
+            bufferCount = 0;
+            waitingCount = 0;
+            maxWaitingCount = 0;
+            invalidReleaseCount = 0;
+        }
     }
 
-    return true;
-}
-
-static void deinitCommonBuffers()
-{
-    if (reorgBuffer)
+    // Get buffer of given size.
+    // Returns nullptr if size is too big. Otherwise may block until buffer is available.
+    // Does not init buffer! Buffer needs to be released with releaseBuffer() after use.
+    void* acquireBuffer(unsigned long long size)
+    {
+        ASSERT(subBufferLock && subBufferPtr);
+#if !defined(NO_UEFI)
+        ASSERT(size <= subBufferSize);
+#endif
+        if (size > subBufferSize)
+            return nullptr;
+
+        // shortcut for default case
+        if (TRY_ACQUIRE(subBufferLock[0]))
+        {
+            return subBufferPtr[0];
+        }
+
+        long cnt = _InterlockedIncrement(&waitingCount);
+        if (maxWaitingCount < cnt)
+            maxWaitingCount = cnt;
+
+        unsigned int i = 0;
+        BEGIN_WAIT_WHILE(TRY_ACQUIRE(subBufferLock[i]) == false)
+        {
+            ++i;
+            if (i >= bufferCount)
+                i = 0;
+        }
+        END_WAIT_WHILE();
+
+        _InterlockedDecrement(&waitingCount);
+
+        return subBufferPtr[i];
+    }
+
+    // Release buffer that was acquired with acquireBuffer() before.
+    void releaseBuffer(void* buffer)
+    {
+        ASSERT(subBufferLock && subBufferPtr && buffer);
+
+        // shortcut for default case
+        if (subBufferPtr[0] == buffer)
+        {
+            if (subBufferLock[0])
+                RELEASE(subBufferLock[0]);
+            else
+                ++invalidReleaseCount;
+            return;
+        }
+
+        // find buffer
+        unsigned int bufferIdx = 1;
+        while (bufferIdx < bufferCount && subBufferPtr[bufferIdx] != buffer)
+            ++bufferIdx;
+
+        // invalid pointer passed?
+#if !defined(NO_UEFI)
+        ASSERT(bufferIdx < bufferCount);
+        ASSERT(subBufferLock[bufferIdx]);
+#endif
+        if (bufferIdx >= bufferCount || !subBufferLock[bufferIdx])
+        {
+            ++invalidReleaseCount;
+            return;
+        }
+
+        // release buffer
+        RELEASE(subBufferLock[bufferIdx]);
+    }
+
+    // Heuristics how many processors were waiting for a buffer in parallel (for deciding the count of buffers)
+    long getMaxWaitingProcessorCount() const
+    {
+        return maxWaitingCount;
+    }
+
+    // Counter of invalid release calls as an indicator if debugging is needed
+    long getInvalidReleaseCount() const
     {
-        freePool(reorgBuffer);
-        reorgBuffer = nullptr;
+        return invalidReleaseCount;
     }
+
+    // Returns number of buffers currently acquired
+    unsigned int acquiredBuffers() const
+    {
+        unsigned int count = 0;
+        for (unsigned int i = 0; i < bufferCount; ++i)
+            if (subBufferLock[i])
+                ++count;
+        return count;
+    }
+
+protected:
+    unsigned char** subBufferPtr = nullptr;
+    volatile char* subBufferLock = nullptr;
+    unsigned long long subBufferSize = 0;
+    unsigned int bufferCount = 0;
+    volatile long waitingCount = 0;
+    long maxWaitingCount = 0;
+    long invalidReleaseCount = 0;
+};
+
+
+GLOBAL_VAR_DECL CommonBuffers commonBuffers;
+
+
+static void* __acquireScratchpad(unsigned long long size, bool initZero = true)
+{
+    void* ptr = commonBuffers.acquireBuffer(size);
+    if (ptr && initZero)
+        setMem(ptr, size, 0);
+    return ptr;
 }
 
-static void* __scratchpad(unsigned long long sizeToMemsetZero)
+static void __releaseScratchpad(void* ptr)
 {
-    ASSERT(sizeToMemsetZero <= reorgBufferSize);
-    if (sizeToMemsetZero)
-        setMem(reorgBuffer, sizeToMemsetZero, 0);
-    return reorgBuffer;
+    commonBuffers.releaseBuffer(ptr);
 }