diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/index.ts b/packages/lodestar-beacon-state-transition/src/fast/block/index.ts index d40e3f0ce090..67b2438e05cb 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/index.ts @@ -1,5 +1,5 @@ -import {EpochContext} from "../util"; -import {BeaconBlock, BeaconState} from "@chainsafe/lodestar-types"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; +import {BeaconBlock} from "@chainsafe/lodestar-types"; import {processBlockHeader} from "./processBlockHeader"; import {processRandao} from "./processRandao"; @@ -25,7 +25,7 @@ export { export function processBlock( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, block: BeaconBlock, verifySignatures = true ): void { diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index 6f94e91373bb..e0c6f90333ec 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -1,25 +1,27 @@ -import {readOnlyMap} from "@chainsafe/ssz"; -import {BeaconState, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; import {FAR_FUTURE_EPOCH} from "../../constants"; import {computeActivationExitEpoch, getChurnLimit} from "../../util"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; /** * Initiate the exit of the validator with index ``index``. */ -export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState, index: ValidatorIndex): void { +export function initiateValidatorExit( + epochCtx: EpochContext, + state: CachedValidatorsBeaconState, + index: ValidatorIndex +): void { const config = epochCtx.config; // return if validator already initiated exit - const validator = state.validators[index]; - if (validator.exitEpoch !== FAR_FUTURE_EPOCH) { + if (state.validators[index].exitEpoch !== FAR_FUTURE_EPOCH) { return; } const currentEpoch = epochCtx.currentShuffling.epoch; // compute exit queue epoch - const validatorExitEpochs = readOnlyMap(state.validators, (v) => v.exitEpoch); + const validatorExitEpochs = state.flatValidators().readOnlyMap((v) => v.exitEpoch); const exitEpochs = validatorExitEpochs.filter((exitEpoch) => exitEpoch !== FAR_FUTURE_EPOCH); exitEpochs.push(computeActivationExitEpoch(config, currentEpoch)); let exitQueueEpoch = Math.max(...exitEpochs); @@ -29,6 +31,8 @@ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState } // set validator exit epoch and withdrawable epoch - validator.exitEpoch = exitQueueEpoch; - validator.withdrawableEpoch = validator.exitEpoch + config.params.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + state.updateValidator(index, { + exitEpoch: exitQueueEpoch, + withdrawableEpoch: exitQueueEpoch + config.params.MIN_VALIDATOR_WITHDRAWABILITY_DELAY, + }); } diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processAttesterSlashing.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processAttesterSlashing.ts index eaa2a13b2f51..b4dfd8df27ee 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processAttesterSlashing.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processAttesterSlashing.ts @@ -1,13 +1,13 @@ -import {AttesterSlashing, BeaconState, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {AttesterSlashing, ValidatorIndex} from "@chainsafe/lodestar-types"; import {isSlashableValidator, isSlashableAttestationData} from "../../util"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; import {slashValidator} from "./slashValidator"; import {isValidIndexedAttestation} from "./isValidIndexedAttestation"; export function processAttesterSlashing( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, attesterSlashing: AttesterSlashing, verifySignatures = true ): void { diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processDeposit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processDeposit.ts index afff5980aa7c..34c466c91b76 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processDeposit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processDeposit.ts @@ -1,12 +1,12 @@ import bls from "@chainsafe/bls"; -import {BeaconState, Deposit} from "@chainsafe/lodestar-types"; +import {Deposit} from "@chainsafe/lodestar-types"; import {verifyMerkleBranch, bigIntMin} from "@chainsafe/lodestar-utils"; import {DEPOSIT_CONTRACT_TREE_DEPTH, DomainType, FAR_FUTURE_EPOCH} from "../../constants"; import {computeDomain, computeSigningRoot, increaseBalance} from "../../util"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; -export function processDeposit(epochCtx: EpochContext, state: BeaconState, deposit: Deposit): void { +export function processDeposit(epochCtx: EpochContext, state: CachedValidatorsBeaconState, deposit: Deposit): void { const config = epochCtx.config; const {EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE} = config.params; // verify the merkle branch @@ -43,7 +43,7 @@ export function processDeposit(epochCtx: EpochContext, state: BeaconState, depos } // add validator and balance entries - state.validators.push({ + state.addValidator({ pubkey: pubkey, withdrawalCredentials: deposit.data.withdrawalCredentials, activationEligibilityEpoch: FAR_FUTURE_EPOCH, diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts index 0259ac8b6678..04a73e5fcd9a 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts @@ -3,13 +3,12 @@ import { Attestation, AttesterSlashing, BeaconBlockBody, - BeaconState, Deposit, ProposerSlashing, VoluntaryExit, } from "@chainsafe/lodestar-types"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; import {processProposerSlashing} from "./processProposerSlashing"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processAttestation} from "./processAttestation"; @@ -17,11 +16,16 @@ import {processDeposit} from "./processDeposit"; import {processVoluntaryExit} from "./processVoluntaryExit"; type Operation = ProposerSlashing | AttesterSlashing | Attestation | Deposit | VoluntaryExit; -type OperationFunction = (epochCtx: EpochContext, state: BeaconState, op: Operation, verify: boolean) => void; +type OperationFunction = ( + epochCtx: EpochContext, + state: CachedValidatorsBeaconState, + op: Operation, + verify: boolean +) => void; export function processOperations( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, body: BeaconBlockBody, verifySignatures = true ): void { diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processProposerSlashing.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processProposerSlashing.ts index 001a11f64539..c3fdc222f918 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processProposerSlashing.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processProposerSlashing.ts @@ -1,13 +1,13 @@ import {BeaconState, ProposerSlashing} from "@chainsafe/lodestar-types"; import {DomainType} from "../../constants"; import {computeEpochAtSlot, computeSigningRoot, getDomain, isSlashableValidator} from "../../util"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; import {slashValidator} from "./slashValidator"; import {ISignatureSet, SignatureSetType, verifySignatureSet} from "../signatureSets"; export function processProposerSlashing( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, proposerSlashing: ProposerSlashing, verifySignatures = true ): void { diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts index 06b7ae44ffd0..48dd5eefe1a9 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts @@ -2,12 +2,12 @@ import {BeaconState, SignedVoluntaryExit} from "@chainsafe/lodestar-types"; import {DomainType, FAR_FUTURE_EPOCH} from "../../constants"; import {computeSigningRoot, getDomain, isActiveValidator} from "../../util"; import {ISignatureSet, SignatureSetType, verifySignatureSet} from "../signatureSets"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; import {initiateValidatorExit} from "./initiateValidatorExit"; export function processVoluntaryExit( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, signedVoluntaryExit: SignedVoluntaryExit, verifySignature = true ): void { diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts b/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts index 42964e7a289a..c873e7e056f8 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts @@ -1,12 +1,12 @@ -import {BeaconState, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; import {decreaseBalance, increaseBalance} from "../../util"; -import {EpochContext} from "../util"; +import {EpochContext, CachedValidatorsBeaconState} from "../util"; import {initiateValidatorExit} from "./initiateValidatorExit"; export function slashValidator( epochCtx: EpochContext, - state: BeaconState, + state: CachedValidatorsBeaconState, slashedIndex: ValidatorIndex, whistleblowerIndex?: ValidatorIndex ): void { @@ -19,8 +19,10 @@ export function slashValidator( const epoch = epochCtx.currentShuffling.epoch; initiateValidatorExit(epochCtx, state, slashedIndex); const validator = state.validators[slashedIndex]; - validator.slashed = true; - validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR); + state.updateValidator(slashedIndex, { + slashed: true, + withdrawableEpoch: Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR), + }); state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effectiveBalance; decreaseBalance(state, slashedIndex, validator.effectiveBalance / BigInt(MIN_SLASHING_PENALTY_QUOTIENT)); diff --git a/packages/lodestar-beacon-state-transition/src/fast/epoch/index.ts b/packages/lodestar-beacon-state-transition/src/fast/epoch/index.ts index 91eacfcb9533..d9f7b0f21451 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/epoch/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/epoch/index.ts @@ -1,6 +1,4 @@ -import {BeaconState} from "@chainsafe/lodestar-types"; - -import {prepareEpochProcessState} from "../util"; +import {CachedValidatorsBeaconState, prepareEpochProcessState} from "../util"; import {StateTransitionEpochContext} from "../util/epochContext"; import {processJustificationAndFinalization} from "./processJustificationAndFinalization"; import {processRewardsAndPenalties} from "./processRewardsAndPenalties"; @@ -20,7 +18,7 @@ export { getAttestationDeltas, }; -export function processEpoch(epochCtx: StateTransitionEpochContext, state: BeaconState): void { +export function processEpoch(epochCtx: StateTransitionEpochContext, state: CachedValidatorsBeaconState): void { const process = prepareEpochProcessState(epochCtx, state); epochCtx.epochProcess = process; processJustificationAndFinalization(epochCtx, process, state); diff --git a/packages/lodestar-beacon-state-transition/src/fast/epoch/processFinalUpdates.ts b/packages/lodestar-beacon-state-transition/src/fast/epoch/processFinalUpdates.ts index c5f8bc600a23..d4a8c013b765 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/epoch/processFinalUpdates.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/epoch/processFinalUpdates.ts @@ -1,11 +1,15 @@ import {readOnlyMap, List} from "@chainsafe/ssz"; -import {BeaconState, Eth1Data, PendingAttestation} from "@chainsafe/lodestar-types"; +import {Eth1Data, PendingAttestation} from "@chainsafe/lodestar-types"; import {bigIntMin, intDiv} from "@chainsafe/lodestar-utils"; import {getRandaoMix} from "../../util"; -import {EpochContext, IEpochProcess} from "../util"; +import {EpochContext, IEpochProcess, CachedValidatorsBeaconState} from "../util"; -export function processFinalUpdates(epochCtx: EpochContext, process: IEpochProcess, state: BeaconState): void { +export function processFinalUpdates( + epochCtx: EpochContext, + process: IEpochProcess, + state: CachedValidatorsBeaconState +): void { const config = epochCtx.config; const currentEpoch = process.currentEpoch; const nextEpoch = currentEpoch + 1; @@ -37,10 +41,9 @@ export function processFinalUpdates(epochCtx: EpochContext, process: IEpochProce const balance = balances[i]; const effectiveBalance = status.validator.effectiveBalance; if (balance + DOWNWARD_THRESHOLD < effectiveBalance || effectiveBalance + UPWARD_THRESHOLD < balance) { - state.validators[i].effectiveBalance = bigIntMin( - balance - (balance % EFFECTIVE_BALANCE_INCREMENT), - MAX_EFFECTIVE_BALANCE - ); + state.updateValidator(i, { + effectiveBalance: bigIntMin(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE), + }); } } diff --git a/packages/lodestar-beacon-state-transition/src/fast/epoch/processRegistryUpdates.ts b/packages/lodestar-beacon-state-transition/src/fast/epoch/processRegistryUpdates.ts index bd121fcfdf6d..5377a257a762 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/epoch/processRegistryUpdates.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/epoch/processRegistryUpdates.ts @@ -1,20 +1,22 @@ -import {BeaconState} from "@chainsafe/lodestar-types"; - import {computeActivationExitEpoch} from "../../util"; -import {EpochContext, IEpochProcess} from "../util"; +import {EpochContext, IEpochProcess, CachedValidatorsBeaconState} from "../util"; -export function processRegistryUpdates(epochCtx: EpochContext, process: IEpochProcess, state: BeaconState): void { +export function processRegistryUpdates( + epochCtx: EpochContext, + process: IEpochProcess, + state: CachedValidatorsBeaconState +): void { const config = epochCtx.config; let exitEnd = process.exitQueueEnd; let endChurn = process.exitQueueEndChurn; const {MIN_VALIDATOR_WITHDRAWABILITY_DELAY} = epochCtx.config.params; // process ejections for (const index of process.indicesToEject) { - const validator = state.validators[index]; - // set validator exit epoch and withdrawable epoch - validator.exitEpoch = exitEnd; - validator.withdrawableEpoch = exitEnd + MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + state.updateValidator(index, { + exitEpoch: exitEnd, + withdrawableEpoch: exitEnd + MIN_VALIDATOR_WITHDRAWABILITY_DELAY, + }); endChurn += 1; if (endChurn >= process.churnLimit) { @@ -25,7 +27,9 @@ export function processRegistryUpdates(epochCtx: EpochContext, process: IEpochPr // set new activation eligibilities for (const index of process.indicesToSetActivationEligibility) { - state.validators[index].activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1; + state.updateValidator(index, { + activationEligibilityEpoch: epochCtx.currentShuffling.epoch + 1, + }); } const finalityEpoch = state.finalizedCheckpoint.epoch; @@ -35,6 +39,8 @@ export function processRegistryUpdates(epochCtx: EpochContext, process: IEpochPr if (process.statuses[index].validator.activationEligibilityEpoch > finalityEpoch) { break; // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early } - state.validators[index].activationEpoch = computeActivationExitEpoch(config, process.currentEpoch); + state.updateValidator(index, { + activationEpoch: computeActivationExitEpoch(config, process.currentEpoch), + }); } } diff --git a/packages/lodestar-beacon-state-transition/src/fast/index.ts b/packages/lodestar-beacon-state-transition/src/fast/index.ts index fd9a5d0fef82..649e8904d060 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/index.ts @@ -1,6 +1,6 @@ -import {BeaconState, SignedBeaconBlock} from "@chainsafe/lodestar-types"; +import {SignedBeaconBlock} from "@chainsafe/lodestar-types"; -import {verifyBlockSignature} from "./util"; +import {CachedValidatorsBeaconState, verifyBlockSignature} from "./util"; import {IStateContext} from "./util"; import {StateTransitionEpochContext} from "./util/epochContext"; import {EpochContext} from "./util/epochContext"; @@ -22,7 +22,7 @@ export function fastStateTransition( const types = epochCtx.config.types; const block = signedBlock.message; - const postState = types.BeaconState.clone(state); + const postState = state.clone(); // process slots (including those with no blocks) since block processSlots(epochCtx, postState, block.slot); @@ -46,7 +46,10 @@ export function fastStateTransition( /** * Trim epochProcess in epochCtx, and insert the standard/exchange interface epochProcess to the final IStateContext */ -export function toIStateContext(epochCtx: StateTransitionEpochContext, state: BeaconState): IStateContext { +export function toIStateContext( + epochCtx: StateTransitionEpochContext, + state: CachedValidatorsBeaconState +): IStateContext { const epochProcess = epochCtx.epochProcess; epochCtx.epochProcess = undefined; return { diff --git a/packages/lodestar-beacon-state-transition/src/fast/slot/index.ts b/packages/lodestar-beacon-state-transition/src/fast/slot/index.ts index 1b66be7fd568..b0886f6c312d 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/slot/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/slot/index.ts @@ -1,12 +1,17 @@ -import {BeaconState, Slot} from "@chainsafe/lodestar-types"; +import {Slot} from "@chainsafe/lodestar-types"; import {StateTransitionEpochContext} from "../util/epochContext"; import {processEpoch} from "../epoch"; import {processSlot} from "./processSlot"; +import {CachedValidatorsBeaconState} from "../util/interface"; export {processSlot}; -export function processSlots(epochCtx: StateTransitionEpochContext, state: BeaconState, slot: Slot): void { +export function processSlots( + epochCtx: StateTransitionEpochContext, + state: CachedValidatorsBeaconState, + slot: Slot +): void { if (!(state.slot < slot)) { throw new Error("State slot must transition to a future slot: " + `stateSlot=${state.slot} slot=${slot}`); } diff --git a/packages/lodestar-beacon-state-transition/src/fast/util/epochContext.ts b/packages/lodestar-beacon-state-transition/src/fast/util/epochContext.ts index 28386b850130..d1fb78e4865a 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/util/epochContext.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/util/epochContext.ts @@ -26,6 +26,7 @@ import { } from "../../util"; import {computeEpochShuffling, IEpochShuffling} from "./epochShuffling"; import {IEpochProcess} from "./epochProcess"; +import {CachedValidatorsBeaconState} from "./interface"; export class PubkeyIndexMap extends Map { get(key: ByteVector): ValidatorIndex | undefined { @@ -138,15 +139,13 @@ export class EpochContext { * Called to re-use information, such as the shuffling of the next epoch, after transitioning into a * new epoch. */ - public rotateEpochs(state: BeaconState): void { + public rotateEpochs(state: CachedValidatorsBeaconState): void { this.previousShuffling = this.currentShuffling; this.currentShuffling = this.nextShuffling; const nextEpoch = this.currentShuffling.epoch + 1; - const indicesBounded: [ValidatorIndex, Epoch, Epoch][] = readOnlyMap(state.validators, (v, i) => [ - i, - v.activationEpoch, - v.exitEpoch, - ]); + const indicesBounded = state + .flatValidators() + .readOnlyMap<[number, Epoch, Epoch]>((v, i) => [i, v.activationEpoch, v.exitEpoch]); this.nextShuffling = computeEpochShuffling(this.config, state, indicesBounded, nextEpoch); this._resetProposers(state); } diff --git a/packages/lodestar-beacon-state-transition/src/fast/util/epochProcess.ts b/packages/lodestar-beacon-state-transition/src/fast/util/epochProcess.ts index 1226edcfeacf..47bc3e804e0b 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/util/epochProcess.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/util/epochProcess.ts @@ -1,5 +1,5 @@ import {List, readOnlyForEach, readOnlyMap} from "@chainsafe/ssz"; -import {Epoch, ValidatorIndex, Gwei, BeaconState, PendingAttestation} from "@chainsafe/lodestar-types"; +import {Epoch, ValidatorIndex, Gwei, PendingAttestation} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import {computeActivationExitEpoch, getBlockRootAtSlot, computeStartSlotAtEpoch, getChurnLimit} from "../../util"; @@ -19,7 +19,8 @@ import { } from "./attesterStatus"; import {IEpochStakeSummary} from "./epochStakeSummary"; import {StateTransitionEpochContext} from "./epochContext"; -import {createIFlatValidator, isActiveIFlatValidator} from "./flatValidator"; +import {isActiveIFlatValidator} from "./flatValidator"; +import {CachedValidatorsBeaconState} from "./interface"; /** * The AttesterStatus (and FlatValidator under status.validator) objects and @@ -67,7 +68,10 @@ export function createIEpochProcess(): IEpochProcess { }; } -export function prepareEpochProcessState(epochCtx: StateTransitionEpochContext, state: BeaconState): IEpochProcess { +export function prepareEpochProcessState( + epochCtx: StateTransitionEpochContext, + state: CachedValidatorsBeaconState +): IEpochProcess { const out = createIEpochProcess(); const config = epochCtx.config; @@ -87,8 +91,7 @@ export function prepareEpochProcessState(epochCtx: StateTransitionEpochContext, let exitQueueEnd = computeActivationExitEpoch(config, currentEpoch); let activeCount = 0; - readOnlyForEach(state.validators, (validator, i) => { - const v = createIFlatValidator(validator); + state.flatValidators().readOnlyForEach((v, i) => { const status = createIAttesterStatus(v); if (v.slashed) { diff --git a/packages/lodestar-beacon-state-transition/src/fast/util/index.ts b/packages/lodestar-beacon-state-transition/src/fast/util/index.ts index c5b3d4840b0b..ebe7c71885b7 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/util/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/util/index.ts @@ -1,6 +1,6 @@ -import {BeaconState} from "@chainsafe/lodestar-types"; import {EpochContext} from "./epochContext"; import {IEpochProcess} from "./epochProcess"; +import {CachedValidatorsBeaconState} from "./interface"; export * from "./block"; export * from "./attesterStatus"; @@ -15,7 +15,7 @@ export * from "./interface"; * Exchange Interface of StateContext */ export interface IStateContext { - state: BeaconState; + state: CachedValidatorsBeaconState; epochCtx: EpochContext; epochProcess?: IEpochProcess; } diff --git a/packages/lodestar-beacon-state-transition/src/fast/util/interface.ts b/packages/lodestar-beacon-state-transition/src/fast/util/interface.ts index 1142e34a0a4c..1d519f477235 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/util/interface.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/util/interface.ts @@ -1,6 +1,9 @@ import {IReadonlyEpochShuffling} from "."; -import {ValidatorIndex, Slot} from "@chainsafe/lodestar-types"; -import {ByteVector} from "@chainsafe/ssz"; +import {ValidatorIndex, Slot, BeaconState, Validator} from "@chainsafe/lodestar-types"; +import {ByteVector, readOnlyForEach} from "@chainsafe/ssz"; +import {createIFlatValidator, IFlatValidator} from "./flatValidator"; +import {config} from "@chainsafe/lodestar-config/lib/presets/mainnet"; +import {Vector} from "@chainsafe/persistent-ts"; /** * Readonly interface for EpochContext. @@ -12,3 +15,122 @@ export type ReadonlyEpochContext = { readonly previousShuffling?: IReadonlyEpochShuffling; getBeaconProposer: (slot: Slot) => ValidatorIndex; }; + +/** + * Cache validators of state using a persistent vector to improve the loop performance. + * Instead of accessing `validators` array directly inside BeaconState, use: + * + flatValidators() for the loop + * + updateValidator() for an update + * + addValidator() for a creation + * that'd update both the cached validators and the one in the original state. + */ +// eslint-disable-next-line @typescript-eslint/naming-convention +export interface CachedValidatorsBeaconState extends BeaconState { + flatValidators(): Vector; + updateValidator(i: ValidatorIndex, value: Partial): void; + addValidator(validator: Validator): void; + getOriginalState(): BeaconState; + clone(): CachedValidatorsBeaconState; +} + +/** + * Looping through validators inside TreeBacked is so expensive. + * Cache validators from TreeBacked using a persistent vector for efficiency. + * When write, write to both the cache and TreeBacked. + * When read, just return the cache. + */ +export class CachedValidatorsBeaconState { + // the original BeaconState + private _state: BeaconState; + // this is immutable and shared across BeaconStates for most of the validators + private _cachedValidators: Vector; + + constructor(state: BeaconState, cachedValidators: Vector) { + this._state = state; + this._cachedValidators = cachedValidators; + } + + public createProxy(): CachedValidatorsBeaconState { + return new Proxy(this, new CachedValidatorsBeaconStateProxyHandler()); + } + + /** + * Write to both the cached validator and BeaconState. + * _cachedValidators refers to a new instance + */ + public updateValidator(i: ValidatorIndex, value: Partial): void { + if (this._cachedValidators) { + const validator = this._cachedValidators.get(i); + this._cachedValidators = this._cachedValidators.set(i, {...validator!, ...value}); + } + const validator = this._state.validators[i]; + Object.assign(validator, value); + } + + /** + * Add validator to both the cache and BeaconState + * _cachedValidators refers to a new instance + */ + public addValidator(validator: Validator): void { + this._cachedValidators = this._cachedValidators.append(createIFlatValidator(validator)); + this._state.validators.push(validator); + } + + /** + * Loop through the cached validators, not the TreeBacked validators inside BeaconState. + */ + public flatValidators(): Vector { + return this._cachedValidators; + } + + /** + * This is very cheap thanks to persistent-merkle-tree and persistent-vector. + */ + public clone(): CachedValidatorsBeaconState { + const clonedState = config.types.BeaconState.clone(this._state); + const clonedCachedValidators = this._cachedValidators.clone(); + return new CachedValidatorsBeaconState(clonedState, clonedCachedValidators).createProxy(); + } + + public getOriginalState(): BeaconState { + return this._state; + } +} + +/** + * Convenient method to create a CachedValidatorsBeaconState from a BeaconState + * @param state + */ +export function createCachedValidatorsBeaconState(state: BeaconState): CachedValidatorsBeaconState { + const tmpValidators: IFlatValidator[] = []; + readOnlyForEach(state.validators, (validator) => { + tmpValidators.push(createIFlatValidator(validator)); + }); + return new CachedValidatorsBeaconState(state, Vector.of(...tmpValidators)).createProxy(); +} + +class CachedValidatorsBeaconStateProxyHandler implements ProxyHandler { + /** + * Forward all BeaconState property getters to _state. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public get(target: CachedValidatorsBeaconState, p: keyof BeaconState): any { + if (target[p] !== undefined) { + return target[p]; + } + return target.getOriginalState()[p]; + } + + /** + * Forward all BeaconState property setters to _state. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public set(target: CachedValidatorsBeaconState, p: keyof BeaconState, value: any): boolean { + if (target[p] !== undefined) { + target[p] = value; + } else { + target.getOriginalState()[p] = value; + } + return true; + } +} diff --git a/packages/lodestar-beacon-state-transition/test/perf/epoch_processing.test.ts b/packages/lodestar-beacon-state-transition/test/perf/epoch_processing.test.ts index eccd903715d5..afca658d3fd2 100644 --- a/packages/lodestar-beacon-state-transition/test/perf/epoch_processing.test.ts +++ b/packages/lodestar-beacon-state-transition/test/perf/epoch_processing.test.ts @@ -1,7 +1,11 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; -import {BeaconState} from "@chainsafe/lodestar-types"; import {WinstonLogger} from "@chainsafe/lodestar-utils"; -import {IEpochProcess, prepareEpochProcessState} from "../../src/fast/util"; +import { + CachedValidatorsBeaconState, + createCachedValidatorsBeaconState, + IEpochProcess, + prepareEpochProcessState, +} from "../../src/fast/util"; import {EpochContext, StateTransitionEpochContext} from "../../src/fast/util/epochContext"; import { processFinalUpdates, @@ -15,7 +19,7 @@ import {generatePerformanceState, initBLS} from "./util"; import {expect} from "chai"; describe("Epoch Processing Performance Tests", function () { - let state: BeaconState; + let state: CachedValidatorsBeaconState; let epochCtx: StateTransitionEpochContext; let process: IEpochProcess; const logger = new WinstonLogger(); @@ -23,11 +27,12 @@ describe("Epoch Processing Performance Tests", function () { before(async function () { this.timeout(0); await initBLS(); - state = await generatePerformanceState(); + const origState = await generatePerformanceState(); // go back 1 slot to process epoch - state.slot -= 1; + origState.slot -= 1; epochCtx = new EpochContext(config); - epochCtx.loadState(state); + epochCtx.loadState(origState); + state = createCachedValidatorsBeaconState(origState); }); it("prepareEpochProcessState", async () => { @@ -36,7 +41,7 @@ describe("Epoch Processing Performance Tests", function () { process = prepareEpochProcessState(epochCtx, state); logger.profile("prepareEpochProcessState"); // not stable, sometimes < 1400, sometimes > 2000 - expect(Date.now() - start).lt(1500); + expect(Date.now() - start).lt(100); }); it("processJustificationAndFinalization", async () => { diff --git a/packages/lodestar-beacon-state-transition/test/perf/sanity/blocks.test.ts b/packages/lodestar-beacon-state-transition/test/perf/sanity/blocks.test.ts index bc506aea889e..15bd9cb2a993 100644 --- a/packages/lodestar-beacon-state-transition/test/perf/sanity/blocks.test.ts +++ b/packages/lodestar-beacon-state-transition/test/perf/sanity/blocks.test.ts @@ -4,6 +4,7 @@ import {WinstonLogger} from "@chainsafe/lodestar-utils"; import {List} from "@chainsafe/ssz"; import {expect} from "chai"; import {EpochContext, fastStateTransition, IStateContext} from "../../../src/fast"; +import {createCachedValidatorsBeaconState} from "../../../src/fast/util"; import {generatePerformanceBlock, generatePerformanceState, initBLS} from "../util"; describe("Process Blocks Performance Test", function () { @@ -12,10 +13,10 @@ describe("Process Blocks Performance Test", function () { const logger = new WinstonLogger(); before(async () => { await initBLS(); - const state = await generatePerformanceState(); + const origState = await generatePerformanceState(); const epochCtx = new EpochContext(config); - epochCtx.loadState(state); - stateCtx = {state, epochCtx}; + epochCtx.loadState(origState); + stateCtx = {state: createCachedValidatorsBeaconState(origState), epochCtx}; }); it("should process block", async () => { @@ -27,7 +28,7 @@ describe("Process Blocks Performance Test", function () { verifySignatures: false, verifyStateRoot: false, }); - expect(Date.now() - start).lt(25); + expect(Date.now() - start).lte(25); logger.profile(`Process block ${signedBlock.message.slot}`); }); @@ -50,8 +51,7 @@ describe("Process Blocks Performance Test", function () { verifySignatures: false, verifyStateRoot: false, }); - // could be up to 7000 - expect(Date.now() - start).lt(6000); + expect(Date.now() - start).lt(200); logger.profile(`Process block ${signedBlock.message.slot} with ${numValidatorExits} validator exits`); }); }); diff --git a/packages/lodestar-beacon-state-transition/test/perf/sanity/slots.test.ts b/packages/lodestar-beacon-state-transition/test/perf/sanity/slots.test.ts index 38dd0ea5772d..d425a9460549 100644 --- a/packages/lodestar-beacon-state-transition/test/perf/sanity/slots.test.ts +++ b/packages/lodestar-beacon-state-transition/test/perf/sanity/slots.test.ts @@ -1,23 +1,27 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; -import {BeaconState} from "@chainsafe/lodestar-types"; import {WinstonLogger} from "@chainsafe/lodestar-utils"; import {expect} from "chai"; import {EpochContext} from "../../../src/fast"; import {processSlots} from "../../../src/fast/slot"; import {StateTransitionEpochContext} from "../../../src/fast/util/epochContext"; +import {CachedValidatorsBeaconState, createCachedValidatorsBeaconState} from "../../../src/fast/util"; import {generatePerformanceState, initBLS} from "../util"; describe("Process Slots Performance Test", function () { this.timeout(0); const logger = new WinstonLogger(); - let state: BeaconState; + let state: CachedValidatorsBeaconState; let epochCtx: StateTransitionEpochContext; before(async () => { await initBLS(); - state = await generatePerformanceState(); + }); + + beforeEach(async () => { + const origState = await generatePerformanceState(); epochCtx = new EpochContext(config); - epochCtx.loadState(state); + epochCtx.loadState(origState); + state = createCachedValidatorsBeaconState(origState); }); it("process 1 empty epoch", async () => { @@ -26,7 +30,7 @@ describe("Process Slots Performance Test", function () { const start = Date.now(); processSlots(epochCtx, state, state.slot + numSlot); logger.profile(`Process ${numSlot} slots`); - expect(Date.now() - start).lt(2800); + expect(Date.now() - start).lt(1100); }); it("process double empty epochs", async () => { @@ -35,7 +39,7 @@ describe("Process Slots Performance Test", function () { const start = Date.now(); processSlots(epochCtx, state, state.slot + numSlot); logger.profile(`Process ${numSlot} slots`); - expect(Date.now() - start).lt(5300); + expect(Date.now() - start).lt(2200); }); it("process 4 empty epochs", async () => { @@ -44,6 +48,6 @@ describe("Process Slots Performance Test", function () { const start = Date.now(); processSlots(epochCtx, state, state.slot + numSlot); logger.profile(`Process ${numSlot} slots`); - expect(Date.now() - start).lt(11000); + expect(Date.now() - start).lt(4300); }); }); diff --git a/packages/lodestar-beacon-state-transition/test/unit/fast/util/interface.test.ts b/packages/lodestar-beacon-state-transition/test/unit/fast/util/interface.test.ts new file mode 100644 index 000000000000..252cbf6669bb --- /dev/null +++ b/packages/lodestar-beacon-state-transition/test/unit/fast/util/interface.test.ts @@ -0,0 +1,83 @@ +import {config} from "@chainsafe/lodestar-config/lib/presets/mainnet"; +import {BeaconState, Validator} from "@chainsafe/lodestar-types"; +import {List, TreeBacked} from "@chainsafe/ssz"; +import {expect} from "chai"; +import {createCachedValidatorsBeaconState, CachedValidatorsBeaconState} from "../../../../src/fast/util"; +import {generateState} from "../../../utils/state"; + +const NUM_VALIDATORS = 100000; + +describe("StateTransitionBeaconState", function () { + let state: TreeBacked; + let wrappedState: CachedValidatorsBeaconState; + + before(function () { + this.timeout(0); + const validators: Validator[] = []; + for (let i = 0; i < NUM_VALIDATORS; i++) { + validators.push({ + pubkey: Buffer.alloc(48), + withdrawalCredentials: Buffer.alloc(32), + effectiveBalance: BigInt(1000000), + slashed: false, + activationEligibilityEpoch: i + 10, + activationEpoch: i, + exitEpoch: i + 20, + withdrawableEpoch: i + 30, + }); + } + const defaultState = generateState({validators: validators as List}); + state = config.types.BeaconState.tree.createValue(defaultState); + }); + + beforeEach(() => { + state = state.clone(); + wrappedState = createCachedValidatorsBeaconState(state); + }); + + it("should read the same value of TreeBacked", () => { + expect(state.validators[1000].activationEpoch).to.be.equal(1000); + expect(wrappedState.validators[1000].activationEpoch).to.be.equal(1000); + expect(wrappedState.flatValidators().get(1000)!.activationEpoch).to.be.equal(1000); + }); + + it("should modify both state and wrappedState", () => { + const oldFlatValidator = wrappedState.flatValidators().get(1000); + wrappedState.updateValidator(1000, {activationEpoch: 2020, exitEpoch: 2030}); + expect(wrappedState.flatValidators().get(1000)!.activationEpoch).to.be.equal(2020); + expect(wrappedState.flatValidators().get(1000)!.exitEpoch).to.be.equal(2030); + // other property is the same + expect(wrappedState.flatValidators().get(1000)!.effectiveBalance).to.be.equal(oldFlatValidator?.effectiveBalance); + expect(wrappedState.flatValidators().get(1000)!.slashed).to.be.equal(oldFlatValidator?.slashed); + expect(wrappedState.flatValidators().get(1000)!.activationEligibilityEpoch).to.be.equal( + oldFlatValidator?.activationEligibilityEpoch + ); + expect(wrappedState.flatValidators().get(1000)!.withdrawableEpoch).to.be.equal(oldFlatValidator?.withdrawableEpoch); + + expect(state.validators[1000].activationEpoch).to.be.equal(2020); + expect(state.validators[1000].exitEpoch).to.be.equal(2030); + // other property is the same + expect(state.validators[1000].effectiveBalance).to.be.equal(oldFlatValidator?.effectiveBalance); + expect(state.validators[1000].slashed).to.be.equal(oldFlatValidator?.slashed); + expect(state.validators[1000].activationEligibilityEpoch).to.be.equal(oldFlatValidator?.activationEligibilityEpoch); + expect(state.validators[1000].withdrawableEpoch).to.be.equal(oldFlatValidator?.withdrawableEpoch); + }); + + it("should add validator to both state and wrappedState", () => { + wrappedState.addValidator({ + pubkey: Buffer.alloc(48), + withdrawalCredentials: Buffer.alloc(32), + effectiveBalance: BigInt(1000000), + slashed: false, + activationEligibilityEpoch: NUM_VALIDATORS + 10, + activationEpoch: NUM_VALIDATORS, + exitEpoch: NUM_VALIDATORS + 20, + withdrawableEpoch: NUM_VALIDATORS + 30, + }); + + expect(wrappedState.flatValidators().length).to.be.equal(NUM_VALIDATORS + 1); + expect(state.validators.length).to.be.equal(NUM_VALIDATORS + 1); + expect(wrappedState.flatValidators().get(NUM_VALIDATORS)!.activationEpoch).to.be.equal(NUM_VALIDATORS); + expect(state.validators[NUM_VALIDATORS].activationEpoch).to.be.equal(NUM_VALIDATORS); + }); +}); diff --git a/packages/lodestar/src/api/impl/validator/validator.ts b/packages/lodestar/src/api/impl/validator/validator.ts index 4b4b6c8af77d..9344c84c93ac 100644 --- a/packages/lodestar/src/api/impl/validator/validator.ts +++ b/packages/lodestar/src/api/impl/validator/validator.ts @@ -10,6 +10,7 @@ import { AttestationData, AttesterDuty, BeaconBlock, + BeaconState, Bytes96, CommitteeIndex, Epoch, @@ -20,7 +21,7 @@ import { ValidatorIndex, } from "@chainsafe/lodestar-types"; import {assert, ILogger} from "@chainsafe/lodestar-utils"; -import {readOnlyForEach} from "@chainsafe/ssz"; +import {readOnlyForEach, TreeBacked} from "@chainsafe/ssz"; import {IAttestationJob, IBeaconChain} from "../../../chain"; import {assembleAttestationData} from "../../../chain/factory/attestation"; import {assembleBlock} from "../../../chain/factory/block"; @@ -85,7 +86,13 @@ export class ValidatorApi implements IValidatorApi { await checkSyncStatus(this.config, this.sync); const headRoot = this.chain.forkChoice.getHeadRoot(); const {state, epochCtx} = await this.chain.regen.getBlockSlotState(headRoot, slot); - return await assembleAttestationData(epochCtx.config, state, headRoot, slot, committeeIndex); + return await assembleAttestationData( + epochCtx.config, + state.getOriginalState() as TreeBacked, + headRoot, + slot, + committeeIndex + ); } catch (e) { this.logger.warn("Failed to produce attestation data", e); throw e; diff --git a/packages/lodestar/src/chain/blocks/stateTransition.ts b/packages/lodestar/src/chain/blocks/stateTransition.ts index 255c7d9f0d48..477a65dd1c66 100644 --- a/packages/lodestar/src/chain/blocks/stateTransition.ts +++ b/packages/lodestar/src/chain/blocks/stateTransition.ts @@ -1,5 +1,5 @@ -import {byteArrayEquals, TreeBacked} from "@chainsafe/ssz"; -import {BeaconState, Slot} from "@chainsafe/lodestar-types"; +import {byteArrayEquals} from "@chainsafe/ssz"; +import {Slot} from "@chainsafe/lodestar-types"; import {assert} from "@chainsafe/lodestar-utils"; import { ZERO_HASH, @@ -23,6 +23,7 @@ import {sleep} from "@chainsafe/lodestar-utils"; import {IBeaconDb} from "../../db"; import {BlockError, BlockErrorCode} from "../errors"; import {verifySignatureSetsBatch} from "../bls"; +import {StateTransitionEpochContext} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/epochContext"; /** * Emits a properly formed "checkpoint" event, given a checkpoint state context @@ -65,26 +66,30 @@ export async function processSlotsToNearestCheckpoint( emitter: ChainEventEmitter, stateCtx: ITreeStateContext, slot: Slot -): Promise { +): Promise { const config = stateCtx.epochCtx.config; const {SLOTS_PER_EPOCH} = config.params; const preSlot = stateCtx.state.slot; const postSlot = slot; const preEpoch = computeEpochAtSlot(config, preSlot); - let postCtx = cloneStateCtx(stateCtx); + const postCtx = cloneStateCtx(stateCtx); + const stateTransitionState = postCtx.state; + const stateTranstionEpochContext = new StateTransitionEpochContext(undefined, postCtx.epochCtx); for ( let nextEpochSlot = computeStartSlotAtEpoch(config, preEpoch + 1); nextEpochSlot <= postSlot; nextEpochSlot += SLOTS_PER_EPOCH ) { - processSlots(postCtx.epochCtx, postCtx.state, nextEpochSlot); - postCtx = toTreeStateContext(toIStateContext(postCtx.epochCtx, postCtx.state)); - emitCheckpointEvent(emitter, postCtx); - postCtx = cloneStateCtx(postCtx); + processSlots(stateTranstionEpochContext, stateTransitionState, nextEpochSlot); + const checkpointCtx = toTreeStateContext(toIStateContext(stateTranstionEpochContext, stateTransitionState)); + emitCheckpointEvent(emitter, cloneStateCtx(checkpointCtx)); // this avoids keeping our node busy processing blocks await sleep(0); } - return postCtx; + return { + epochCtx: stateTranstionEpochContext, + state: stateTransitionState, + }; } /** @@ -97,12 +102,11 @@ export async function processSlotsByCheckpoint( stateCtx: ITreeStateContext, slot: Slot ): Promise { - let postCtx = await processSlotsToNearestCheckpoint(emitter, stateCtx, slot); + const postCtx = await processSlotsToNearestCheckpoint(emitter, stateCtx, slot); if (postCtx.state.slot < slot) { processSlots(postCtx.epochCtx, postCtx.state, slot); - postCtx = toTreeStateContext(toIStateContext(postCtx.epochCtx, postCtx.state)); } - return postCtx; + return toTreeStateContext(postCtx); } export function emitForkChoiceHeadEvents( @@ -175,7 +179,7 @@ export async function runStateTransition( // it should always have epochBalances there bc it's a checkpoint state, ie got through processEpoch const justifiedBalances = (await db.checkpointStateCache.get(postStateContext.state.currentJustifiedCheckpoint)) ?.epochCtx.epochBalances; - forkChoice.onBlock(job.signedBlock.message, postStateContext.state, justifiedBalances); + forkChoice.onBlock(job.signedBlock.message, postStateContext.state.getOriginalState(), justifiedBalances); if (postSlot % SLOTS_PER_EPOCH === 0) { emitCheckpointEvent(emitter, postStateContext); @@ -196,7 +200,7 @@ export async function runStateTransition( */ function toTreeStateContext(stateCtx: IStateContext): ITreeStateContext { const treeStateCtx: ITreeStateContext = { - state: stateCtx.state as TreeBacked, + state: stateCtx.state, epochCtx: new LodestarEpochContext(undefined, stateCtx.epochCtx), }; diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index 4aaf93ea7629..b33a9f4948d7 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -148,8 +148,8 @@ export class BeaconChain implements IBeaconChain { return headStateRoot; } public async getHeadState(): Promise> { - // head state should always have epoch ctx - return (await this.getHeadStateContext()).state; + //head state should always have epoch ctx + return (await this.getHeadStateContext()).state.getOriginalState() as TreeBacked; } public async getHeadEpochContext(): Promise { // head should always have epoch ctx diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index b9836db1b896..7a318b5cc4f3 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -219,7 +219,7 @@ export async function onBlock( job: IBlockJob ): Promise { const blockRoot = this.config.types.BeaconBlock.hashTreeRoot(block.message); - this.logger.debug("Block processed", { + this.logger.info("Block processed", { slot: block.message.slot, root: toHexString(blockRoot), }); @@ -309,7 +309,7 @@ export async function onErrorBlock(this: BeaconChain, err: BlockError): Promise< return; } - this.logger.debug("Block error", {}, err); + this.logger.error("Block error", {}, err); const blockRoot = this.config.types.BeaconBlock.hashTreeRoot(err.job.signedBlock.message); switch (err.type.code) { diff --git a/packages/lodestar/src/chain/factory/block/index.ts b/packages/lodestar/src/chain/factory/block/index.ts index 740ad4c8edb9..fea1a90475b7 100644 --- a/packages/lodestar/src/chain/factory/block/index.ts +++ b/packages/lodestar/src/chain/factory/block/index.ts @@ -4,7 +4,8 @@ import {processBlock} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import {BeaconBlock, Bytes96, Root, Slot} from "@chainsafe/lodestar-types"; +import {BeaconBlock, BeaconState, Bytes96, Root, Slot} from "@chainsafe/lodestar-types"; +import {TreeBacked} from "@chainsafe/ssz"; import {ZERO_HASH} from "../../../constants"; import {IBeaconDb} from "../../../db/api"; import {ITreeStateContext} from "../../../db/api/beacon/stateContextCache"; @@ -29,7 +30,14 @@ export async function assembleBlock( proposerIndex: stateContext.epochCtx.getBeaconProposer(slot), parentRoot: head.blockRoot, stateRoot: ZERO_HASH, - body: await assembleBody(config, db, eth1, stateContext.state, randaoReveal, graffiti), + body: await assembleBody( + config, + db, + eth1, + stateContext.state.getOriginalState() as TreeBacked, + randaoReveal, + graffiti + ), }; block.stateRoot = computeNewStateRoot(config, stateContext, block); diff --git a/packages/lodestar/src/chain/initState.ts b/packages/lodestar/src/chain/initState.ts index 66e5717ab4e6..2ce90a9c01d5 100644 --- a/packages/lodestar/src/chain/initState.ts +++ b/packages/lodestar/src/chain/initState.ts @@ -16,6 +16,7 @@ import {Eth1Provider} from "../eth1"; import {IBeaconMetrics} from "../metrics"; import {GenesisBuilder} from "./genesis/genesis"; import {IGenesisResult} from "./genesis/interface"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; export async function persistGenesisResult( db: IBeaconDb, @@ -138,7 +139,7 @@ export async function restoreStateCaches( const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const stateCtx = {state, epochCtx}; + const stateCtx = {state: createCachedValidatorsBeaconState(state), epochCtx}; await Promise.all([db.stateCache.add(stateCtx), db.checkpointStateCache.add(checkpoint, stateCtx)]); } diff --git a/packages/lodestar/src/db/api/beacon/stateContextCache.ts b/packages/lodestar/src/db/api/beacon/stateContextCache.ts index 2bd9dcbcc085..1df91a7f19f1 100644 --- a/packages/lodestar/src/db/api/beacon/stateContextCache.ts +++ b/packages/lodestar/src/db/api/beacon/stateContextCache.ts @@ -1,10 +1,11 @@ import {ByteVector, toHexString, TreeBacked} from "@chainsafe/ssz"; import {BeaconState, Gwei} from "@chainsafe/lodestar-types"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; // Lodestar specifc state context export interface ITreeStateContext { - state: TreeBacked; + state: CachedValidatorsBeaconState; // TreeBacked epochCtx: LodestarEpochContext; } @@ -40,7 +41,9 @@ export class StateContextCache { } public async add(item: ITreeStateContext): Promise { - this.cache[toHexString(item.state.hashTreeRoot())] = this.clone(item); + this.cache[toHexString((item.state.getOriginalState() as TreeBacked).hashTreeRoot())] = this.clone( + item + ); } public async delete(root: ByteVector): Promise { diff --git a/packages/lodestar/src/tasks/tasks/archiveStates.ts b/packages/lodestar/src/tasks/tasks/archiveStates.ts index 2c6217089f58..a1776ef2d310 100644 --- a/packages/lodestar/src/tasks/tasks/archiveStates.ts +++ b/packages/lodestar/src/tasks/tasks/archiveStates.ts @@ -6,7 +6,8 @@ import {ITask} from "../interface"; import {IBeaconDb} from "../../db/api"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {Checkpoint} from "@chainsafe/lodestar-types"; +import {BeaconState, Checkpoint} from "@chainsafe/lodestar-types"; +import {TreeBacked} from "@chainsafe/ssz"; export interface IArchiveStatesModules { db: IBeaconDb; @@ -41,7 +42,7 @@ export class ArchiveStatesTask implements ITask { throw Error("No state in cache for finalized checkpoint state epoch #" + this.finalized.epoch); } const finalizedState = stateCache.state; - await this.db.stateArchive.put(finalizedState.slot, finalizedState); + await this.db.stateArchive.put(finalizedState.slot, finalizedState.getOriginalState() as TreeBacked); // don't delete states before the finalized state, auto-prune will take care of it this.logger.info("Archive states completed", {finalizedEpoch: this.finalized.epoch}); this.logger.profile("Archive States epoch #" + this.finalized.epoch); diff --git a/packages/lodestar/test/e2e/chain/factory/block/assembleBlock.test.ts b/packages/lodestar/test/e2e/chain/factory/block/assembleBlock.test.ts index 2bb25a7861fc..74673ee47cb3 100644 --- a/packages/lodestar/test/e2e/chain/factory/block/assembleBlock.test.ts +++ b/packages/lodestar/test/e2e/chain/factory/block/assembleBlock.test.ts @@ -27,6 +27,7 @@ import {ValidatorApi} from "../../../../../src/api/impl/validator"; import {StubbedBeaconDb} from "../../../../utils/stub"; import {silentLogger} from "../../../../utils/logger"; import {StateRegenerator} from "../../../../../src/chain/regen"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describe("produce block", function () { this.timeout("10 min"); @@ -71,7 +72,9 @@ describe("produce block", function () { sinon.stub(epochCtx, "getBeaconProposer").returns(20); const slotState = state.clone(); slotState.slot = 1; - regenStub.getBlockSlotState.withArgs(sinon.match.any, 1).resolves({state: slotState, epochCtx}); + regenStub.getBlockSlotState + .withArgs(sinon.match.any, 1) + .resolves({state: createCachedValidatorsBeaconState(slotState), epochCtx}); forkChoiceStub.getHead.returns(parentBlockSummary); dbStub.depositDataRoot.getTreeBacked.resolves(depositDataRootList); dbStub.proposerSlashing.values.resolves([]); @@ -90,7 +93,8 @@ describe("produce block", function () { return await assembleBlock(config, chainStub, dbStub, eth1, slot, randao); }); const block = await blockProposingService.createAndPublishBlock(0, 1, state.fork, ZERO_HASH); - expect(() => fastStateTransition({state, epochCtx}, block!, {verifyStateRoot: false})).to.not.throw(); + const wrappedState = createCachedValidatorsBeaconState(state); + expect(() => fastStateTransition({state: wrappedState, epochCtx}, block!, {verifyStateRoot: false})).to.not.throw(); }); function getBlockProposingService(secretKey: SecretKey): BlockProposingService { diff --git a/packages/lodestar/test/unit/api/impl/beacon/state/utils.test.ts b/packages/lodestar/test/unit/api/impl/beacon/state/utils.test.ts index 108368d1660a..37d7fc7ec7c0 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/state/utils.test.ts @@ -12,7 +12,7 @@ import {getEpochBeaconCommittees, resolveStateId} from "../../../../../../src/ap import {BeaconChain, IBeaconChain} from "../../../../../../src/chain"; import {IBeaconClock} from "../../../../../../src/chain/clock/interface"; import {generateBlockSummary} from "../../../../../utils/block"; -import {generateState} from "../../../../../utils/state"; +import {generateCachedState, generateState} from "../../../../../utils/state"; import {StubbedBeaconDb} from "../../../../../utils/stub"; import {generateValidator} from "../../../../../utils/validator"; @@ -30,7 +30,7 @@ describe("beacon state api utils", function () { it("resolve head state id - success", async function () { forkChoiceStub.getHead.returns(generateBlockSummary({stateRoot: Buffer.alloc(32, 1)})); - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, "head"); expect(state).to.not.be.null; expect(forkChoiceStub.getHead.calledOnce).to.be.true; @@ -46,7 +46,7 @@ describe("beacon state api utils", function () { it("resolve finalized state id - success", async function () { forkChoiceStub.getFinalizedCheckpoint.returns({root: Buffer.alloc(32, 1), epoch: 1}); - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, "finalized"); expect(state).to.not.be.null; expect(forkChoiceStub.getFinalizedCheckpoint.calledOnce).to.be.true; @@ -64,7 +64,7 @@ describe("beacon state api utils", function () { it("resolve justified state id - success", async function () { forkChoiceStub.getJustifiedCheckpoint.returns({root: Buffer.alloc(32, 1), epoch: 1}); - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, "justified"); expect(state).to.not.be.null; expect(forkChoiceStub.getJustifiedCheckpoint.calledOnce).to.be.true; @@ -81,14 +81,14 @@ describe("beacon state api utils", function () { }); it("resolve state by root", async function () { - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, toHexString(Buffer.alloc(32, 1))); expect(state).to.not.be.null; expect(dbStub.stateCache.get.calledOnce).to.be.true; }); it.skip("resolve finalized state by root", async function () { - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, toHexString(Buffer.alloc(32, 1))); expect(state).to.be.null; expect(dbStub.stateCache.get.calledOnce).to.be.true; @@ -103,7 +103,7 @@ describe("beacon state api utils", function () { forkChoiceStub.getCanonicalBlockSummaryAtSlot .withArgs(123) .returns(generateBlockSummary({stateRoot: Buffer.alloc(32, 1)})); - dbStub.stateCache.get.resolves({state: generateState(), epochCtx: null!}); + dbStub.stateCache.get.resolves({state: generateCachedState(), epochCtx: null!}); const state = await resolveStateId(config, dbStub, forkChoiceStub, "123"); expect(state).to.not.be.null; expect(forkChoiceStub.getCanonicalBlockSummaryAtSlot.withArgs(123).calledOnce).to.be.true; @@ -165,7 +165,7 @@ describe("beacon state api utils", function () { function getApiContext(): ApiStateContext { return { - state: generateState(), + state: generateCachedState(), epochCtx: { currentShuffling: { committees: [ diff --git a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts index 462caa5cf722..4892a43edbd1 100644 --- a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts @@ -13,6 +13,7 @@ import {generateState} from "../../../../../utils/state"; import {StubbedBeaconDb} from "../../../../../utils/stub"; import {BeaconSync, IBeaconSync} from "../../../../../../src/sync"; import {generateValidators} from "../../../../../utils/validator"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describe("get proposers api impl", function () { const sandbox = sinon.createSandbox(); @@ -79,7 +80,7 @@ describe("get proposers api impl", function () { const epochCtx = new EpochContext(config); epochCtx.loadState(state); chainStub.getHeadStateContextAtCurrentEpoch.resolves({ - state, + state: createCachedValidatorsBeaconState(state), epochCtx, }); sinon.stub(epochCtx, "getBeaconProposer").returns(1); diff --git a/packages/lodestar/test/unit/chain/attestation/process.test.ts b/packages/lodestar/test/unit/chain/attestation/process.test.ts index 13215c1c7781..acafab189f8f 100644 --- a/packages/lodestar/test/unit/chain/attestation/process.test.ts +++ b/packages/lodestar/test/unit/chain/attestation/process.test.ts @@ -11,7 +11,7 @@ import {ChainEvent, ChainEventEmitter} from "../../../../src/chain"; import {StateRegenerator} from "../../../../src/chain/regen"; import {AttestationErrorCode} from "../../../../src/chain/errors"; import {generateAttestation} from "../../../utils/attestation"; -import {generateState} from "../../../utils/state"; +import {generateCachedState} from "../../../utils/state"; describe("processAttestation", function () { const emitter = new ChainEventEmitter(); @@ -47,7 +47,7 @@ describe("processAttestation", function () { it("should throw on errored getIndexedAttestation", async function () { const attestation = generateAttestation(); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.getIndexedAttestation.throws(); regen.getCheckpointState.resolves({state, epochCtx: (epochCtx as unknown) as EpochContext}); @@ -66,7 +66,7 @@ describe("processAttestation", function () { it("should throw on invalid indexed attestation", async function () { const attestation = generateAttestation(); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.getIndexedAttestation.returns({} as IndexedAttestation); regen.getCheckpointState.resolves({state, epochCtx: (epochCtx as unknown) as EpochContext}); @@ -86,7 +86,7 @@ describe("processAttestation", function () { it("should emit 'attestation' event on processed attestation", async function () { const attestation = generateAttestation(); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.getIndexedAttestation.returns({} as IndexedAttestation); regen.getCheckpointState.resolves({state, epochCtx: (epochCtx as unknown) as EpochContext}); diff --git a/packages/lodestar/test/unit/chain/chain.test.ts b/packages/lodestar/test/unit/chain/chain.test.ts index f66565dfb9db..b8f2905ce07a 100644 --- a/packages/lodestar/test/unit/chain/chain.test.ts +++ b/packages/lodestar/test/unit/chain/chain.test.ts @@ -1,7 +1,6 @@ import {expect} from "chai"; import sinon from "sinon"; -import {TreeBacked} from "@chainsafe/ssz"; import {BeaconState} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/minimal"; import {bytesToInt, WinstonLogger} from "@chainsafe/lodestar-utils"; @@ -14,6 +13,7 @@ import {generateBlockSummary} from "../../utils/block"; import {generateState} from "../../utils/state"; import {StubbedBeaconDb} from "../../utils/stub"; import {generateValidators} from "../../utils/validator"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describe("BeaconChain", function () { const sandbox = sinon.createSandbox(); @@ -26,7 +26,10 @@ describe("BeaconChain", function () { metrics = new BeaconMetrics({enabled: false} as any, {logger}); const state: BeaconState = generateState(); state.validators = generateValidators(5, {activationEpoch: 0}); - dbStub.stateCache.get.resolves({state: state as TreeBacked, epochCtx: new EpochContext(config)}); + dbStub.stateCache.get.resolves({ + state: createCachedValidatorsBeaconState(state), + epochCtx: new EpochContext(config), + }); dbStub.stateArchive.lastValue.resolves(state as any); chain = new BeaconChain({opts: defaultChainOptions, config, db: dbStub, logger, metrics, anchorState: state}); }); diff --git a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts index e845f3376dae..d3fa41bb3861 100644 --- a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts +++ b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts @@ -13,7 +13,7 @@ import * as blockBodyAssembly from "../../../../../src/chain/factory/block/body" import {StateRegenerator} from "../../../../../src/chain/regen"; import {Eth1ForBlockProduction} from "../../../../../src/eth1/"; import {generateBlockSummary, generateEmptyBlock} from "../../../../utils/block"; -import {generateState} from "../../../../utils/state"; +import {generateCachedState, generateState} from "../../../../utils/state"; import {StubbedBeaconDb, StubbedChain} from "../../../../utils/stub"; describe("block assembly", function () { @@ -48,7 +48,7 @@ describe("block assembly", function () { const epochCtx = new EpochContext(config); sinon.stub(epochCtx).getBeaconProposer.returns(2); regenStub.getBlockSlotState.resolves({ - state: generateState({slot: 1}), + state: generateCachedState({slot: 1}), epochCtx: epochCtx, }); beaconDB.depositDataRoot.getTreeBacked.resolves(config.types.DepositDataRootList.tree.defaultValue()); diff --git a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts index b9227598361c..ddf1740924e1 100644 --- a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts @@ -16,7 +16,7 @@ import {validateGossipAggregateAndProof} from "../../../../src/chain/validation" import {ATTESTATION_PROPAGATION_SLOT_RANGE, MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../../../src/constants"; import * as validationUtils from "../../../../src/chain/validation/utils"; import {generateSignedAggregateAndProof} from "../../../utils/aggregateAndProof"; -import {generateState} from "../../../utils/state"; +import {generateCachedState} from "../../../utils/state"; import {StubbedBeaconDb} from "../../../utils/stub"; import {AttestationErrorCode} from "../../../../src/chain/errors"; @@ -196,7 +196,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); regen.getBlockSlotState.resolves({ state, @@ -226,7 +226,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); regen.getBlockSlotState.resolves({ state, @@ -254,7 +254,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.index2pubkey = []; const privateKey = bls.SecretKey.fromBytes(bigIntToBytes(BigInt(1), 32)); @@ -286,7 +286,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.index2pubkey = []; const privateKey = bls.SecretKey.fromBytes(bigIntToBytes(BigInt(1), 32)); @@ -327,7 +327,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.index2pubkey = []; const privateKey = bls.SecretKey.fromBytes(bigIntToBytes(BigInt(1), 32)); @@ -361,7 +361,7 @@ describe("gossip aggregate and proof test", function () { }, }, }); - const state = generateState(); + const state = generateCachedState(); const epochCtx = sinon.createStubInstance(EpochContext); epochCtx.index2pubkey = []; const privateKey = bls.SecretKey.fromKeygen(); diff --git a/packages/lodestar/test/unit/chain/validation/attestation.test.ts b/packages/lodestar/test/unit/chain/validation/attestation.test.ts index 4a03cafae8bf..4a9634998d51 100644 --- a/packages/lodestar/test/unit/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/unit/chain/validation/attestation.test.ts @@ -16,7 +16,7 @@ import {IStateRegenerator, StateRegenerator} from "../../../../src/chain/regen"; import {ATTESTATION_PROPAGATION_SLOT_RANGE} from "../../../../src/constants"; import {validateGossipAttestation} from "../../../../src/chain/validation"; import {generateAttestation} from "../../../utils/attestation"; -import {generateState} from "../../../utils/state"; +import {generateCachedState} from "../../../utils/state"; import {LocalClock} from "../../../../src/chain/clock"; import {IEpochShuffling} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/epochShuffling"; import {AttestationErrorCode} from "../../../../src/chain/errors"; @@ -235,7 +235,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; regen.getCheckpointState.resolves(attestationPreState); @@ -265,7 +265,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.getIndexedAttestation = () => toIndexedAttestation(attestation); @@ -300,7 +300,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -338,7 +338,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -379,7 +379,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -423,7 +423,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -485,7 +485,7 @@ describe("gossip attestation validation", function () { db.seenAttestationCache.hasCommitteeAttestation.resolves(false); forkChoice.hasBlock.returns(true); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -525,7 +525,7 @@ describe("gossip attestation validation", function () { forkChoice.isDescendant.returns(true); forkChoice.isDescendantOfFinalized.returns(false); const attestationPreState = { - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }; attestationPreState.epochCtx.previousShuffling = { @@ -557,7 +557,7 @@ describe("gossip attestation validation", function () { const attestation = generateAttestation({ aggregationBits: [true] as BitList, }); - const state = generateState(); + const state = generateCachedState(); const attestationPreState = { state, epochCtx: new EpochContext(config), diff --git a/packages/lodestar/test/unit/chain/validation/block.test.ts b/packages/lodestar/test/unit/chain/validation/block.test.ts index 6b373d095516..3a7359465c6a 100644 --- a/packages/lodestar/test/unit/chain/validation/block.test.ts +++ b/packages/lodestar/test/unit/chain/validation/block.test.ts @@ -11,7 +11,7 @@ import {StateRegenerator} from "../../../../src/chain/regen"; import {validateGossipBlock} from "../../../../src/chain/validation"; import {generateSignedBlock, getNewBlockJob} from "../../../utils/block"; import {StubbedBeaconDb} from "../../../utils/stub"; -import {generateState} from "../../../utils/state"; +import {generateCachedState} from "../../../utils/state"; import {BlockErrorCode} from "../../../../src/chain/errors"; describe("gossip block validation", function () { @@ -122,7 +122,7 @@ describe("gossip block validation", function () { dbStub.badBlock.has.resolves(false); dbStub.block.get.resolves(null); regenStub.getBlockSlotState.resolves({ - state: generateState(), + state: generateCachedState(), epochCtx: new EpochContext(config), }); verifySignatureStub.returns(false); @@ -150,7 +150,7 @@ describe("gossip block validation", function () { dbStub.block.get.resolves(null); const epochCtxStub = sinon.createStubInstance(EpochContext); regenStub.getBlockSlotState.resolves({ - state: generateState(), + state: generateCachedState(), epochCtx: (epochCtxStub as unknown) as EpochContext, }); verifySignatureStub.returns(true); @@ -181,7 +181,7 @@ describe("gossip block validation", function () { forkChoiceStub.isDescendantOfFinalized.returns(true); const epochCtxStub = sinon.createStubInstance(EpochContext); regenStub.getBlockSlotState.resolves({ - state: generateState(), + state: generateCachedState(), epochCtx: (epochCtxStub as unknown) as EpochContext, }); verifySignatureStub.returns(true); diff --git a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts index 7b5d5f527d12..ad3ca0be4210 100644 --- a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts +++ b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts @@ -15,6 +15,7 @@ import {generateState} from "../../../utils/state"; import {generateEmptySignedVoluntaryExit} from "../../../utils/attestation"; import {validateGossipVoluntaryExit} from "../../../../src/chain/validation/voluntaryExit"; import {VoluntaryExitErrorCode} from "../../../../src/chain/errors/voluntaryExitError"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describe("validate voluntary exit", () => { const sandbox = sinon.createSandbox(); @@ -48,7 +49,7 @@ describe("validate voluntary exit", () => { }); const epochCtx = new EpochContext(config); epochCtx.loadState(state); - regenStub.getCheckpointState.resolves({state, epochCtx}); + regenStub.getCheckpointState.resolves({state: createCachedValidatorsBeaconState(state), epochCtx}); try { await validateGossipVoluntaryExit(config, chainStub, dbStub, voluntaryExit); } catch (error) { @@ -69,7 +70,7 @@ describe("validate voluntary exit", () => { }); const epochCtx = new EpochContext(config); epochCtx.loadState(state); - regenStub.getCheckpointState.resolves({state, epochCtx}); + regenStub.getCheckpointState.resolves({state: createCachedValidatorsBeaconState(state), epochCtx}); isValidIncomingVoluntaryExitStub.returns(false); try { await validateGossipVoluntaryExit(config, chainStub, dbStub, voluntaryExit); @@ -91,7 +92,7 @@ describe("validate voluntary exit", () => { }); const epochCtx = new EpochContext(config); epochCtx.loadState(state); - regenStub.getCheckpointState.resolves({state, epochCtx}); + regenStub.getCheckpointState.resolves({state: createCachedValidatorsBeaconState(state), epochCtx}); isValidIncomingVoluntaryExitStub.returns(true); const validationTest = await validateGossipVoluntaryExit(config, chainStub, dbStub, voluntaryExit); expect(validationTest).to.not.throw; diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index eeb666da89a4..a5a74880ea1b 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -26,6 +26,7 @@ import {IStateRegenerator, StateRegenerator} from "../../../../src/chain/regen"; import {StubbedBeaconDb} from "../../stub"; import {BlockPool} from "../../../../src/chain/blocks"; import {AttestationPool} from "../../../../src/chain/attestation"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; export interface IMockChainParams { genesisTime: Number64; @@ -82,21 +83,21 @@ export class MockBeaconChain implements IBeaconChain { public async getHeadStateContext(): Promise { return { - state: this.state!, + state: createCachedValidatorsBeaconState(this.state!), epochCtx: new EpochContext(this.config), }; } public async getHeadStateContextAtCurrentEpoch(): Promise { return { - state: this.state!, + state: createCachedValidatorsBeaconState(this.state!), epochCtx: new EpochContext(this.config), }; } public async getHeadStateContextAtCurrentSlot(): Promise { return { - state: this.state!, + state: createCachedValidatorsBeaconState(this.state!), epochCtx: new EpochContext(this.config), }; } @@ -112,7 +113,7 @@ export class MockBeaconChain implements IBeaconChain { } public async getHeadState(): Promise> { - return (await this.getHeadStateContext()).state; + return (await this.getHeadStateContext()).state.getOriginalState() as TreeBacked; } public async getUnfinalizedBlocksAtSlots(slots: Slot[]): Promise { diff --git a/packages/lodestar/test/utils/state.ts b/packages/lodestar/test/utils/state.ts index 6dce5a74052d..0692567685b4 100644 --- a/packages/lodestar/test/utils/state.ts +++ b/packages/lodestar/test/utils/state.ts @@ -3,6 +3,10 @@ import {GENESIS_EPOCH, GENESIS_SLOT, ZERO_HASH} from "../../src/constants"; import {generateEmptyBlock} from "./block"; import {config} from "@chainsafe/lodestar-config/minimal"; import {TreeBacked, List} from "@chainsafe/ssz"; +import { + CachedValidatorsBeaconState, + createCachedValidatorsBeaconState, +} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. @@ -75,3 +79,7 @@ export function generateState(opts: TestBeaconState = {}): TreeBacked(); + +// (1, 2, 3, 4) +List.of(1, 2, 3, 4); + +// (1, 2) +List.of(1).prepend(2); + +// 1 +List.of(1, 2, 3).head(); + +// (2, 3) +List.of(1, 2, 3).tail(); + +// (1, 2, 3) +List.of(1, 2, 3, 4).take(3); + +// (4) +List.of(1, 2, 3, 4).drop(3); + +//[1, 2, 3, 4] +[...List.of(1, 2, 3, 4)]; +``` + +### Vector + +`Vector` is a Radix Tree in the vein of clojure's data structure. +This can be used as an immutable sequence with efficient random access and +appending. Here's a sample of its operations: + +```ts +// [] +Vector.empty(); + +// [1] +Vector.empty().append(1); + +// [] +Vector.of(1).pop(); + +// 3 +Vector.of(1, 2, 3).get(2); + +// [1, 2, 100] +Vector.of(1, 2, 3).set(2, 100); +``` diff --git a/packages/persistent-ts/package.json b/packages/persistent-ts/package.json new file mode 100644 index 000000000000..5944e6b0bb5c --- /dev/null +++ b/packages/persistent-ts/package.json @@ -0,0 +1,54 @@ +{ + "name": "@chainsafe/persistent-ts", + "version": "0.13.0", + "description": "Persistent data structures for TypeScript.", + "main": "lib/index.js", + "files": [ + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "scripts": { + "build": "concurrently \"yarn build:lib\" \"yarn build:types\"", + "build:lib": "babel src -x .ts -d lib --source-maps", + "build:lib:watch": "yarn run build:lib --watch", + "build:release": "yarn clean && yarn build", + "build:types": "tsc --incremental --declaration --outDir lib --project tsconfig.build.json --emitDeclarationOnly", + "build:types:watch": "yarn run build:types --watch --preserveWatchOutput", + "check-types": "tsc --noEmit", + "clean": "rm -rf lib && rm -f tsconfig.tsbuildinfo && rm -f tsconfig.build.tsbuildinfo", + "lint": "eslint --color --ext .ts src/ test/", + "lint:fix": "yarn run lint --fix", + "prepublishOnly": "yarn build", + "test:unit": "mocha 'test/unit/**/*.test.ts'", + "test:perf": "mocha 'test/perf/**/*.test.ts'" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/cronokirby/persistent-ts.git" + }, + "keywords": [ + "persistent", + "functional", + "typescript" + ], + "author": "Lúcás Meier", + "license": "MIT", + "bugs": { + "url": "https://github.com/cronokirby/persistent-ts/issues" + }, + "homepage": "https://github.com/cronokirby/persistent-ts#readme", + "devDependencies": { + "fast-check": "^1.15.1" + }, + "jest": { + "transform": { + ".ts": "ts-jest" + }, + "testRegex": "\\.test.ts$", + "moduleFileExtensions": [ + "ts", + "js" + ] + } +} diff --git a/packages/persistent-ts/src/List.ts b/packages/persistent-ts/src/List.ts new file mode 100644 index 000000000000..c84f0653ab0f --- /dev/null +++ b/packages/persistent-ts/src/List.ts @@ -0,0 +1,197 @@ +type Node = {next: null} | {value: T; next: Node}; +// We can share a single empty node between all lists. +const EMPTY_NODE = {next: null}; + +/** + * List represents an immutable list containing values of type T. + * + * This class is implemented as a singly linked-list, with all the caveats involved. + * + * This class is best used when many values need to be stored and then consumed + * linearly in a first-in-last-out fashion. If direct indexing or quick storing + * at the front and back is needed, then a list isn't the best choice. + * + * Because a List is Iterable, you can loop over it using `for of` and use the spread operator. + */ +export class List implements Iterable { + // eslint-disable-next-line @typescript-eslint/naming-convention + private constructor(private readonly _node: Node) {} + + /** + * O(1) Create a new empty list. + */ + public static empty(): List { + return new List(EMPTY_NODE as Node); + } + + /** + * O(N) Create a list from an array of values. + * + * @param values an array of values the list will contain, in the same order + */ + public static of(...values: T[]): List { + let ret = List.empty(); + for (let i = values.length - 1; i >= 0; --i) { + ret = ret.prepend(values[i]); + } + return ret; + } + + /** + * Check whether or not a list is empty. + * + * This is equivalent to checking if a list has no elements. + */ + public isEmpty(): boolean { + return !this._node.next; + } + + /** + * O(1) Add a new value to the front of the list. + * + * @param value the value to add to the front of the list + */ + public prepend(value: T): List { + return new List({value, next: this._node}); + } + + /** + * O(1) Get the value at the front of the list, if it exists. + * + * This function will return null if `isEmpty()` returns + * true, or if the value at the front of the list happens to be + * `null`. Because of this, be careful when storing values that might + * be `null` inside the list, because this function may return `null` + * even if the list isn't empty. + */ + public head(): T | null { + return this._node.next ? this._node.value : null; + } + + /** + * O(1) Return a list containing the values past the head of the list. + * + * For example: `List.of(1, 2).tail()` gives `List.of(2)`. + * + * If the list is empty, this method returns an empty list. + * + * `l.tail().prepend(l.head())` will always be `l` for any non-empty list `l`. + */ + public tail(): List { + return this._node.next ? new List(this._node.next) : this; + } + + /** + * O(amount) Take a certain number of elements from the front of a List. + * + * If the amount is 0, and empty list is returned. Negative numbers are treated + * the same way. + * + * If the list has less than the amount taken, the entire list is taken. + * + * @param amount the number of elements to take from the front of the list + */ + public take(amount: number): List { + if (amount <= 0 || !this._node.next) return List.empty(); + const base: Node = { + value: this._node.value, + next: EMPTY_NODE as Node, + }; + let latest = base; + let list = this.tail(); + for (let i = 1; i < amount; ++i) { + // We check specifically against empty in case a value is null inside a list + if (list.isEmpty()) break; + const next: Node = { + value: list.head() as T, + next: EMPTY_NODE as Node, + }; + latest.next = next; + latest = next; + list = list.tail(); + } + return new List(base); + } + + /** + * O(amount) Return a list with `amount` elements removed from the front. + * + * If `amount` is greater than or equal to the size of the list, + * an empty list is returned. + * + * If `amount` is less than or equal to 0, the list is returned without modification. + * + * `l.drop(1)` is always equal to `l.tail()`. + * + * @param amount the number of elements to drop + */ + public drop(amount: number): List { + // eslint-disable-next-line @typescript-eslint/no-this-alias + let list: List = this; + for (let i = 0; i < amount; ++i) { + if (list.isEmpty()) break; + list = list.tail(); + } + return list; + } + + /** + * O(Nthis) Concatenate this list and another. + * + * This returns a list containing all the elements in `this` followed + * by all the elements in `that`. + * + * This could be done via `List.of(...this, ...that)` but this would + * copy the elements of both lists, whereas this implementation only + * needs to copy elements from the first list. + * + * @param that the list to append to this list + */ + public concat(that: List): List { + if (!this._node.next) return that; + const base: Node = { + value: this._node.value, + next: that._node, + }; + let latest = base; + let cursor = this._node.next; + while (cursor.next) { + const next: Node = { + value: cursor.value as T, + next: that._node, + }; + latest.next = next; + latest = next; + cursor = cursor.next; + } + return new List(base); + } + + public *[Symbol.iterator](): Iterator { + let node = this._node; + while (node.next) { + yield node.value; + node = node.next; + } + } + + /** + * O(N) Test whether or not a list is logically equal to another. + * + * This returns true if the lists have the same size, and each element in a given + * position is `===` to the element in the same position in the other list. + * + * @param that the list to compare for equality with this one. + */ + public equals(that: List): boolean { + let thisNode = this._node; + let thatNode = that._node; + while (thisNode.next) { + if (!thatNode.next) return false; + if (thisNode.value !== thatNode.value) return false; + thisNode = thisNode.next; + thatNode = thatNode.next; + } + return !thatNode.next; + } +} diff --git a/packages/persistent-ts/src/Vector.ts b/packages/persistent-ts/src/Vector.ts new file mode 100644 index 000000000000..d4a1d645be7c --- /dev/null +++ b/packages/persistent-ts/src/Vector.ts @@ -0,0 +1,334 @@ +const BIT_WIDTH = 5; +const BIT_MASK = 0b11111; +const BRANCH_SIZE = 1 << BIT_WIDTH; +const DEFAULT_LEVEL_SHIFT = 5; + +function isFullBranch(length: number): boolean { + return ( + // initially we initialize Vector with an empty branch (DEFAULT_LEVEL_SHIFT) + // length === 1 << 5 || + length === 1 << 10 || length === 1 << 15 || length === 1 << 20 || length === 1 << 25 || length === 1 << 30 + ); +} + +interface ILeaf { + leaf: true; + values: T[]; +} +interface IBranch { + leaf: false; + // We have explicit nulls because when popping we can null old branches on purpose. + nodes: (INode | null)[]; +} +type INode = ILeaf | IBranch; + +function emptyBranch(): IBranch { + return {leaf: false, nodes: Array(BRANCH_SIZE).fill(null)}; +} + +function emptyLeaf(): ILeaf { + return {leaf: true, values: Array(BRANCH_SIZE).fill(null)}; +} + +function copyNode(vnode: INode): INode { + if (vnode.leaf) { + return {leaf: true, values: [...vnode.values]}; + } else { + return {leaf: false, nodes: [...vnode.nodes]}; + } +} + +function copyBranch(vnode: IBranch): IBranch { + return {leaf: false, nodes: [...vnode.nodes]}; +} + +/** + * The main class. + */ +export class Vector implements Iterable { + private constructor( + private readonly root: IBranch, + private readonly levelShift: number, + private readonly tail: T[], + public readonly length: number + ) {} + + /** + * Create an empty vector of a certain type. + */ + public static empty(): Vector { + return new Vector(emptyBranch(), DEFAULT_LEVEL_SHIFT, Array(BRANCH_SIZE).fill(null), 0); + } + + /** + * Create a new vector containing certain elements. + * + * @param values the values that this vector will contain + */ + public static of(...values: T[]): Vector { + let acc = Vector.empty(); + for (const v of values) acc = acc.append(v); + return acc; + } + + /** + * O(log_32(N)) Return the value at a certain index, if it exists. + * + * This returns null if the index is out of the vector's bounds. + * + * @param index the index to look up + */ + public get(index: number): T | null { + if (index < 0 || index >= this.length) return null; + if (index >= this.getTailOffset()) { + return this.tail[index % BRANCH_SIZE]; + } + let shift = this.levelShift; + let cursor: INode = this.root; + while (!cursor.leaf) { + // This cast is fine because we checked the length prior + cursor = cursor.nodes[(index >>> shift) & BIT_MASK] as INode; + shift -= BIT_WIDTH; + } + return cursor.values[index & BIT_MASK]; + } + + /** + * O(log_32(N)) Return a new vector with an element set to a new value. + * + * This will do nothing if the index is negative, or out of the bounds of the vector. + * + * @param index the index to set + * @param value the value to set at that index + */ + public set(index: number, value: T): Vector { + if (index < 0 || index >= this.length) return this; + if (index >= this.getTailOffset()) { + const newTail = [...this.tail]; + newTail[index & BIT_MASK] = value; + // root is not changed + return new Vector(this.root, this.levelShift, newTail, this.length); + } + const base = copyBranch(this.root); + let shift = this.levelShift; + let cursor: INode = base; + while (!cursor.leaf) { + const subIndex = (index >>> shift) & BIT_MASK; + // This cast is fine because we checked the length prior + const next: INode = copyNode(cursor.nodes[subIndex] as INode); + cursor.nodes[subIndex] = next; + cursor = next; + shift -= BIT_WIDTH; + } + cursor.values[index & BIT_MASK] = value; + // tail is not changed + return new Vector(base, this.levelShift, this.tail, this.length); + } + + /** + * O(log_32(N)) Append a value to the end of this vector. + * + * This is useful for building up a vector from values. + * + * @param value the value to push to the end of the vector + */ + public append(value: T): Vector { + if (this.length - this.getTailOffset() < BRANCH_SIZE) { + // has space in tail + const newTail = [...this.tail]; + newTail[this.length % BRANCH_SIZE] = value; + // root is not changed + return new Vector(this.root, this.levelShift, newTail, this.length + 1); + } + let base: IBranch; + let levelShift = this.levelShift; + if (isFullBranch(this.length - BRANCH_SIZE)) { + base = emptyBranch(); + base.nodes[0] = this.root; + levelShift += BIT_WIDTH; + } else { + base = copyBranch(this.root); + } + // getTailOffset is actually the 1st item in tail + // we now move it to the tree + const index = this.getTailOffset(); + let shift = levelShift; + let cursor: INode = base; + while (!cursor.leaf) { + const subIndex = (index >>> shift) & BIT_MASK; + shift -= BIT_WIDTH; + let next: INode | null = cursor.nodes[subIndex]; + if (!next) { + next = shift === 0 ? emptyLeaf() : emptyBranch(); + } else { + next = copyNode(next); + } + cursor.nodes[subIndex] = next; + cursor = next; + } + // it's safe to update cursor bc "next" is a new instance anyway + cursor.values = [...this.tail]; + return new Vector(base, levelShift, [value, ...Array(BRANCH_SIZE - 1).fill(null)], this.length + 1); + } + + /** + * Return a new Vector with the last element removed. + * + * This does nothing if the Vector contains no elements. + */ + public pop(): Vector { + if (this.length === 0) return this; + // we always have a non-empty tail + const tailLength = this.getTailLength(); + if (tailLength >= 2) { + // ignore the last item + const newTailLength = (this.length - 1) % BRANCH_SIZE; + const newTail = [...this.tail.slice(0, newTailLength), ...Array(BRANCH_SIZE - newTailLength).fill(null)]; + return new Vector(this.root, this.levelShift, newTail, this.length - 1); + } + // tail has exactly 1 item, promote the right most leaf node as tail + const lastItemIndexInTree = this.getTailOffset() - 1; + // Tree has no item + if (lastItemIndexInTree < 0) { + return Vector.empty(); + } + const base = copyBranch(this.root); + let shift = this.levelShift; + let cursor: INode = base; + // we always have a parent bc we create an empty branch initially + let parent: INode | null = null; + let subIndex: number | null = null; + while (!cursor.leaf) { + subIndex = (lastItemIndexInTree >>> shift) & BIT_MASK; + // This cast is fine because we checked the length prior + const next: INode = copyNode(cursor.nodes[subIndex] as INode); + cursor.nodes[subIndex] = next; + parent = cursor; + cursor = next; + shift -= BIT_WIDTH; + } + const newTail = [...cursor.values]; + parent!.nodes[subIndex!] = emptyLeaf(); + let newLevelShift = this.levelShift; + let newRoot: IBranch = base; + if (isFullBranch(this.length - 1 - BRANCH_SIZE)) { + newRoot = base.nodes[0] as IBranch; + newLevelShift -= BIT_WIDTH; + } + return new Vector(copyBranch(newRoot), newLevelShift, newTail, this.length - 1); + } + + /** + * Implement Iterable interface. + */ + public *[Symbol.iterator](): Generator { + let toYield = this.getTailOffset(); + function* iterNode(node: INode): Generator { + if (node.leaf) { + for (const v of node.values) { + if (toYield <= 0) break; + yield v; + --toYield; + } + } else { + for (const next of node.nodes) { + // This check also assures us that the link won't be null + if (toYield <= 0) break; + yield* iterNode(next as INode); + } + } + } + yield* iterNode(this.root); + const tailLength = this.getTailLength(); + for (let i = 0; i < tailLength; i++) yield this.tail[i]; + } + + /** + * Faster way to loop than the regular loop above. + * Same to iterator function but this doesn't yield to improve performance. + */ + public readOnlyForEach(func: (t: T, i: number) => void): void { + let index = 0; + const tailOffset = this.getTailOffset(); + const iterNode = (node: INode): void => { + if (node.leaf) { + for (const v of node.values) { + if (index < tailOffset) { + func(v, index); + index++; + } else { + break; + } + } + } else { + for (const next of node.nodes) { + if (index >= tailOffset) break; + iterNode(next as INode); + } + } + }; + iterNode(this.root); + const tailLength = this.getTailLength(); + for (let i = 0; i < tailLength; i++) { + const value = this.tail[i]; + func(value, index); + index++; + } + } + + /** + * Map to an array of T2. + */ + public readOnlyMap(func: (t: T, i: number) => T2): T2[] { + const result: T2[] = []; + let index = 0; + const tailOffset = this.getTailOffset(); + const iterNode = (node: INode): void => { + if (node.leaf) { + for (const v of node.values) { + if (index < tailOffset) { + result.push(func(v, index)); + index++; + } else { + break; + } + } + } else { + for (const next of node.nodes) { + if (index >= tailOffset) break; + iterNode(next as INode); + } + } + }; + iterNode(this.root); + const tailLength = this.getTailLength(); + for (let i = 0; i < tailLength; i++) { + const value = this.tail[i]; + result.push(func(value, index)); + index++; + } + return result; + } + + /** + * Convert to regular typescript array + */ + public toTS(): Array { + return this.readOnlyMap((v) => v); + } + + /** + * Clone to a new vector. + */ + public clone(): Vector { + return new Vector(this.root, this.levelShift, this.tail, this.length); + } + + private getTailLength(): number { + return this.length - this.getTailOffset(); + } + + private getTailOffset(): number { + return this.length < BRANCH_SIZE ? 0 : ((this.length - 1) >>> BIT_WIDTH) << BIT_WIDTH; + } +} diff --git a/packages/persistent-ts/src/index.ts b/packages/persistent-ts/src/index.ts new file mode 100644 index 000000000000..1c6659d58219 --- /dev/null +++ b/packages/persistent-ts/src/index.ts @@ -0,0 +1,2 @@ +export * from "./List"; +export * from "./Vector"; diff --git a/packages/persistent-ts/test/perf/Vector.test.ts b/packages/persistent-ts/test/perf/Vector.test.ts new file mode 100644 index 000000000000..8b69f798c0d7 --- /dev/null +++ b/packages/persistent-ts/test/perf/Vector.test.ts @@ -0,0 +1,47 @@ +import {expect} from "chai"; +import {Vector} from "../../src/Vector"; + +it("should be able to handle 10M elements", function () { + this.timeout(0); + let start = Date.now(); + let acc = Vector.empty(); + const times = 10000000; + for (let i = 0; i < times; ++i) { + acc = acc.append(i); + } + expect(acc.length).to.be.equal(times); + console.log(`Finish append ${times} items in`, Date.now() - start); + start = Date.now(); + let index = 0; + for (const _ of acc) { + // expect(item).to.be.equal(index); + index++; + } + expect(index).to.be.equal(times); + console.log(`Finish regular iterator ${times} in`, Date.now() - start); + // start = Date.now(); + // for (let i = 0; i < times; ++i) { + // expect(acc.get(i)).to.be.equal(i); + // } + // console.log(`Finish regular for of ${times} items in`, Date.now() - start); + start = Date.now(); + let count = 0; + acc.readOnlyForEach(() => { + count++; + }); + expect(count).to.be.equal(times); + console.log(`Finish readOnlyForEach of ${times} items in`, Date.now() - start); + start = Date.now(); + const tsArray = acc.toTS(); + expect(tsArray.length).to.be.equal(times); + console.log(`Finish toTS of ${times} items in`, Date.now() - start); + start = Date.now(); + const newArr = acc.readOnlyMap((v) => v * 2); + console.log(`Finish readOnlyMap of ${times} items in`, Date.now() - start); + expect(newArr[1]).to.be.equal(2); + expect(newArr.length).to.be.equal(times); + start = Date.now(); + const newArr2 = tsArray.map((v) => v * 2); + console.log(`Finish regular map of regular array of ${times} items in`, Date.now() - start); + expect(newArr).to.be.deep.equal(newArr2); +}); diff --git a/packages/persistent-ts/test/unit/List.test.ts b/packages/persistent-ts/test/unit/List.test.ts new file mode 100644 index 000000000000..ce8cfe4fbc24 --- /dev/null +++ b/packages/persistent-ts/test/unit/List.test.ts @@ -0,0 +1,90 @@ +import fc from "fast-check"; +import {expect} from "chai"; +import {List} from "../../src/List"; + +it("List.empty isEmpty", () => { + const empty = List.empty(); + expect(empty.isEmpty()).to.be.true; +}); + +it("List.singleton is not Empty", () => { + const singleton = List.of(1); + expect(singleton.isEmpty()).to.be.false; +}); + +it("List.equals works", () => { + const empty = List.empty(); + const single1 = List.of(1); + const single2 = List.of(2); + expect(single1.equals(empty)).to.be.false; + expect(single1.equals(single2)).to.be.false; + expect(empty.equals(empty)).to.be.true; + expect(single1.equals(single1)).to.be.true; +}); + +it("List.prepend works", () => { + const single1 = List.of(1); + const prepend1 = List.empty().prepend(1); + expect(prepend1.equals(single1)).to.be.true; + expect(single1.prepend(1).equals(single1)).to.be.false; +}); + +it("List is iterable", () => { + const array = [1, 2, 3]; + const list = List.of(...array); + expect(Array.from(list)).to.be.deep.equal(array); + expect(Array.from(List.empty())).to.be.deep.equal([]); + expect(List.of(...list).equals(list)).to.be.true; +}); + +it("List.head works", () => { + expect(List.empty().head()).to.be.null; + expect(List.of(1).head()).to.be.equal(1); +}); + +it("List.tail works", () => { + const empty = List.empty(); + expect(empty.tail().equals(empty)).to.be.true; +}); + +it("List.take works", () => { + const empty = List.empty(); + const simple = List.of(1, 2, 3); + expect(simple.take(0).equals(empty)).to.be.true; + expect(simple.take(0).equals(simple)).to.be.false; + expect(simple.take(3).equals(simple)).to.be.true; + expect(simple.take(1).equals(List.of(1))).to.be.true; +}); + +it("List.drop works", () => { + const empty = List.empty(); + expect(empty.drop(0).equals(empty)).to.be.true; + expect(empty.drop(1).equals(empty)).to.be.true; + const simple = List.of(1, 2, 3); + expect(simple.drop(3).equals(empty)).to.be.true; + expect(simple.drop(1).equals(List.of(2, 3))).to.be.true; +}); + +it("List.prepend properties", () => { + fc.assert( + fc.property(fc.array(fc.integer()), (data) => { + const list = List.of(...data); + const value = 1; + const extra = list.prepend(value); + expect(extra.tail().equals(list)).to.be.true; + expect(extra.head()).to.be.equal(value); + }) + ); +}); + +it("List can be reassembled from take and drop", () => { + const gen = fc.tuple(fc.array(fc.integer()), fc.nat()); + fc.assert( + fc.property(gen, ([items, amount]) => { + const list = List.of(items); + const left = list.take(amount); + const right = list.drop(amount); + expect(left.concat(right).equals(list)).to.be.true; + }) + ); +}); diff --git a/packages/persistent-ts/test/unit/Vector.test.ts b/packages/persistent-ts/test/unit/Vector.test.ts new file mode 100644 index 000000000000..26c01e0ac71c --- /dev/null +++ b/packages/persistent-ts/test/unit/Vector.test.ts @@ -0,0 +1,141 @@ +import fc from "fast-check"; +import {expect} from "chai"; +import {Vector} from "../../src/Vector"; + +it("Vector.empty has a length of 0", () => { + const empty = Vector.empty(); + expect(empty.length).to.be.equal(0); +}); + +it("Vector.append increments the length", () => { + const empty = Vector.empty(); + expect(empty.append(1).length).to.be.equal(1); + expect(empty.append(1).append(2).length).to.be.equal(2); +}); + +it("Vector.append works with many elements", () => { + let acc = Vector.empty(); + const times = 1025; + for (let i = 0; i < times; ++i) { + acc = acc.append(i); + } + expect(acc.length).to.be.equal(times); + for (let i = 0; i < times; ++i) { + expect(acc.get(i)).to.be.equal(i); + } + let i = 0; + for (const item of acc) { + expect(item).to.be.equal(i); + i++; + } + expect(i).to.be.equal(times); +}); + +it("Vector iterator should work fine", () => { + const times = 1025; + const originalArr = Array.from({length: times}, (_, i) => 2 * i); + const acc = Vector.of(...originalArr); + expect(acc.length).to.be.equal(times); + let i = 0; + for (const item of acc) { + expect(item).to.be.equal(2 * i); + i++; + } + expect(i).to.be.equal(times); + const newArr = [...acc]; + expect(newArr).to.be.deep.equal(originalArr); +}); + +it("Vector readOnlyForEach should work fine", () => { + let acc = Vector.empty(); + const times = 1025; + for (let i = 0; i < times; ++i) { + acc = acc.append(2 * i); + } + expect(acc.length).to.be.equal(times); + let count = 0; + acc.readOnlyForEach((v, i) => { + expect(v).to.be.equal(2 * i); + count++; + }); + expect(count).to.be.equal(times); +}); + +it("Vector readOnlyMap should work fine", () => { + const times = 1025; + const originalArr = Array.from({length: times}, (_, i) => i); + const newArr = originalArr.map((v) => v * 2); + const acc = Vector.of(...originalArr); + expect(acc.length).to.be.equal(times); + const newArr2 = acc.readOnlyMap((v) => v * 2); + expect(newArr2).to.be.deep.equal(newArr); +}); + +it("Vector toTS should convert to regular javascript array", () => { + const times = 1025; + const originalArr = Array.from({length: times}, (_, i) => i); + const acc = Vector.of(...originalArr); + expect(acc.toTS()).to.be.deep.equal(originalArr); +}); + +it("Vector.get works", () => { + const element = 1; + const empty = Vector.empty(); + const single = empty.append(element); + expect(single.get(-1)).to.be.null; + expect(single.get(1)).to.be.null; + expect(empty.get(0)).to.be.null; + expect(single.get(0)).to.be.equal(element); +}); + +it("Vector.set works", () => { + const a = 0; + const b = 1; + const empty = Vector.empty(); + const single = empty.append(a); + expect(single.set(0, b).get(0)).to.be.equal(b); +}); + +it("Vector.set should not effect original vector", () => { + const times = 1025; + const originalArr = Array.from({length: times}, (_, i) => 2 * i); + const originalVector = Vector.of(...originalArr); + let newVector: Vector = originalVector; + for (let i = 0; i < times; i++) { + newVector = newVector.set(i, i * 4); + } + for (let i = 0; i < times; i++) { + expect(newVector!.get(i)).to.be.equal(originalVector.get(i)! * 2); + } + expect([...newVector]).to.be.deep.equal(originalArr.map((item) => item * 2)); + expect([...newVector].length).to.be.equal(1025); + expect(newVector.length).to.be.equal(1025); +}); + +it("Vector.pop works with many elements", () => { + let acc = Vector.empty(); + expect(acc.pop()).to.be.equal(acc); + const times = 1025; + for (let i = 0; i < 2 * times; ++i) { + acc = acc.append(i); + } + for (let i = 0; i < times; ++i) { + acc = acc.pop(); + } + expect(acc.length).to.be.equal(times); + for (let i = 0; i < times; ++i) { + const g = acc.get(i); + expect(g).to.be.equal(i); + } +}); + +it("A Vector created from an array will spread to the same array", () => { + fc.assert( + fc.property(fc.array(fc.integer()), (data) => { + let acc = Vector.empty(); + for (const d of data) acc = acc.append(d); + const arr = [...acc]; + expect(arr).to.be.deep.equal(data); + }) + ); +}); diff --git a/packages/persistent-ts/tsconfig.build.json b/packages/persistent-ts/tsconfig.build.json new file mode 100644 index 000000000000..da29c4d101e8 --- /dev/null +++ b/packages/persistent-ts/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig", + "include": ["src"], + "compilerOptions": { + "typeRoots": ["../../node_modules/@types"], + "outDir": "lib", + "rootDir": "./src" + } +} diff --git a/packages/persistent-ts/tsconfig.json b/packages/persistent-ts/tsconfig.json new file mode 100644 index 000000000000..19e7ebbaeec9 --- /dev/null +++ b/packages/persistent-ts/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig", + "include": ["src", "test"], + "compilerOptions": { + "typeRoots": ["../../node_modules/@types"], + "outDir": "lib", + /* Redirect output structure to the directory. */ + "rootDirs": ["./src", "./test"] + } +} diff --git a/packages/spec-test-runner/test/spec/epoch_processing/finalUpdates/final_updates_fast.test.ts b/packages/spec-test-runner/test/spec/epoch_processing/finalUpdates/final_updates_fast.test.ts index c48642325213..d5c70fb2ef1f 100644 --- a/packages/spec-test-runner/test/spec/epoch_processing/finalUpdates/final_updates_fast.test.ts +++ b/packages/spec-test-runner/test/spec/epoch_processing/finalUpdates/final_updates_fast.test.ts @@ -5,6 +5,7 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processFinalUpdates} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {BeaconState} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IStateTestCase} from "../../../utils/specTestTypes/stateTestCase"; @@ -17,8 +18,9 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); - processFinalUpdates(epochCtx, process, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); + processFinalUpdates(epochCtx, process, wrappedState); return state; }, { diff --git a/packages/spec-test-runner/test/spec/epoch_processing/justification/justification_and_finalization_fast.test.ts b/packages/spec-test-runner/test/spec/epoch_processing/justification/justification_and_finalization_fast.test.ts index fc736bf48228..038595b0219e 100644 --- a/packages/spec-test-runner/test/spec/epoch_processing/justification/justification_and_finalization_fast.test.ts +++ b/packages/spec-test-runner/test/spec/epoch_processing/justification/justification_and_finalization_fast.test.ts @@ -5,6 +5,8 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processJustificationAndFinalization} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; + import {BeaconState} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IStateTestCase} from "../../../utils/specTestTypes/stateTestCase"; @@ -17,8 +19,9 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); - processJustificationAndFinalization(epochCtx, process, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); + processJustificationAndFinalization(epochCtx, process, wrappedState); return state; }, { diff --git a/packages/spec-test-runner/test/spec/epoch_processing/registryUpdates/registry_updates_fast.test.ts b/packages/spec-test-runner/test/spec/epoch_processing/registryUpdates/registry_updates_fast.test.ts index dab2863754d5..3c8b80f398c3 100644 --- a/packages/spec-test-runner/test/spec/epoch_processing/registryUpdates/registry_updates_fast.test.ts +++ b/packages/spec-test-runner/test/spec/epoch_processing/registryUpdates/registry_updates_fast.test.ts @@ -5,6 +5,7 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processRegistryUpdates} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {BeaconState} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IStateTestCase} from "../../../utils/specTestTypes/stateTestCase"; @@ -17,8 +18,9 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); - processRegistryUpdates(epochCtx, process, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); + processRegistryUpdates(epochCtx, process, wrappedState); return state; }, { diff --git a/packages/spec-test-runner/test/spec/epoch_processing/rewardsAndPenalties/rewards_and_penalties_fast.test.ts b/packages/spec-test-runner/test/spec/epoch_processing/rewardsAndPenalties/rewards_and_penalties_fast.test.ts index 79ec9dd004fc..47af8bf8976b 100644 --- a/packages/spec-test-runner/test/spec/epoch_processing/rewardsAndPenalties/rewards_and_penalties_fast.test.ts +++ b/packages/spec-test-runner/test/spec/epoch_processing/rewardsAndPenalties/rewards_and_penalties_fast.test.ts @@ -5,6 +5,7 @@ import {config} from "@chainsafe/lodestar-config/minimal"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processRewardsAndPenalties} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {BeaconState} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IStateTestCase} from "../../../utils/specTestTypes/stateTestCase"; @@ -17,8 +18,9 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); - processRewardsAndPenalties(epochCtx, process, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); + processRewardsAndPenalties(epochCtx, process, wrappedState); return state; }, { diff --git a/packages/spec-test-runner/test/spec/epoch_processing/slashings/slashings_fast.test.ts b/packages/spec-test-runner/test/spec/epoch_processing/slashings/slashings_fast.test.ts index 5886b102e9f8..12958793541c 100644 --- a/packages/spec-test-runner/test/spec/epoch_processing/slashings/slashings_fast.test.ts +++ b/packages/spec-test-runner/test/spec/epoch_processing/slashings/slashings_fast.test.ts @@ -6,6 +6,7 @@ import {BeaconState} from "@chainsafe/lodestar-types"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processSlashings} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IStateTestCase} from "../../../utils/specTestTypes/stateTestCase"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -17,8 +18,9 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); - processSlashings(epochCtx, process, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); + processSlashings(epochCtx, process, wrappedState); return state; }, { diff --git a/packages/spec-test-runner/test/spec/finality/finality_fast.test.ts b/packages/spec-test-runner/test/spec/finality/finality_fast.test.ts index 965118d18536..75064bf7efc0 100644 --- a/packages/spec-test-runner/test/spec/finality/finality_fast.test.ts +++ b/packages/spec-test-runner/test/spec/finality/finality_fast.test.ts @@ -7,16 +7,18 @@ import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-tes import {IFinalityTestCase} from "./type"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {SPEC_TEST_LOCATION} from "../../utils/specTestCases"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describeDirectorySpecTest( "finality fast", join(SPEC_TEST_LOCATION, "/tests/mainnet/phase0/finality/finality/pyspec_tests"), (testcase) => { const state = config.types.BeaconState.tree.createValue(testcase.pre); + const wrappedState = createCachedValidatorsBeaconState(state); const epochCtx = new EpochContext(config); epochCtx.loadState(state); const verify = !!testcase.meta && !!testcase.meta.blsSetting && testcase.meta.blsSetting === BigInt(1); - let stateContext: IStateContext = {epochCtx, state}; + let stateContext: IStateContext = {epochCtx, state: wrappedState}; for (let i = 0; i < Number(testcase.meta.blocksCount); i++) { stateContext = fastStateTransition(stateContext, testcase[`blocks_${i}`] as SignedBeaconBlock, { verifyStateRoot: verify, @@ -24,7 +26,7 @@ describeDirectorySpecTest( verifySignatures: verify, }); } - return stateContext.state; + return stateContext.state.getOriginalState(); }, { inputTypes: { diff --git a/packages/spec-test-runner/test/spec/operations/attesterSlashing/attester_slashing_fast.test.ts b/packages/spec-test-runner/test/spec/operations/attesterSlashing/attester_slashing_fast.test.ts index 1cfd74a4a1ee..f4c2574ce47c 100644 --- a/packages/spec-test-runner/test/spec/operations/attesterSlashing/attester_slashing_fast.test.ts +++ b/packages/spec-test-runner/test/spec/operations/attesterSlashing/attester_slashing_fast.test.ts @@ -4,6 +4,7 @@ import {BeaconState} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processAttesterSlashing} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessAttesterSlashingTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -16,7 +17,8 @@ describeDirectorySpecTest( const epochCtx = new EpochContext(config); epochCtx.loadState(state); const verify = !!testcase.meta && !!testcase.meta.blsSetting && testcase.meta.blsSetting === BigInt(1); - processAttesterSlashing(epochCtx, state, testcase.attester_slashing, verify); + const wrappedState = createCachedValidatorsBeaconState(state); + processAttesterSlashing(epochCtx, wrappedState, testcase.attester_slashing, verify); return state; }, { diff --git a/packages/spec-test-runner/test/spec/operations/deposit/deposit_fast.test.ts b/packages/spec-test-runner/test/spec/operations/deposit/deposit_fast.test.ts index 670644febcbe..384b83019b39 100644 --- a/packages/spec-test-runner/test/spec/operations/deposit/deposit_fast.test.ts +++ b/packages/spec-test-runner/test/spec/operations/deposit/deposit_fast.test.ts @@ -4,6 +4,7 @@ import {BeaconState} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processDeposit} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessDepositTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -15,7 +16,8 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - processDeposit(epochCtx, state, testcase.deposit); + const wrappedState = createCachedValidatorsBeaconState(state); + processDeposit(epochCtx, wrappedState, testcase.deposit); return state; }, { diff --git a/packages/spec-test-runner/test/spec/operations/proposerSlashing/proposer_slashing_fast.test.ts b/packages/spec-test-runner/test/spec/operations/proposerSlashing/proposer_slashing_fast.test.ts index 3ffbb0c8f7b2..ba310f7ba542 100644 --- a/packages/spec-test-runner/test/spec/operations/proposerSlashing/proposer_slashing_fast.test.ts +++ b/packages/spec-test-runner/test/spec/operations/proposerSlashing/proposer_slashing_fast.test.ts @@ -4,6 +4,7 @@ import {BeaconState} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processProposerSlashing} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessProposerSlashingTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -15,7 +16,8 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - processProposerSlashing(epochCtx, state, testcase.proposer_slashing); + const wrappedState = createCachedValidatorsBeaconState(state); + processProposerSlashing(epochCtx, wrappedState, testcase.proposer_slashing); return state; }, { diff --git a/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts b/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts index 8b24f86a0ec7..1cdc44834637 100644 --- a/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts +++ b/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts @@ -4,6 +4,7 @@ import {BeaconState} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processVoluntaryExit} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessVoluntaryExitTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -15,7 +16,8 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - processVoluntaryExit(epochCtx, state, testcase.voluntary_exit); + const wrappedState = createCachedValidatorsBeaconState(state); + processVoluntaryExit(epochCtx, wrappedState, testcase.voluntary_exit); return state; }, { diff --git a/packages/spec-test-runner/test/spec/rewards/rewards_fast.test.ts b/packages/spec-test-runner/test/spec/rewards/rewards_fast.test.ts index d577bca695e5..e2a75d0c7e02 100644 --- a/packages/spec-test-runner/test/spec/rewards/rewards_fast.test.ts +++ b/packages/spec-test-runner/test/spec/rewards/rewards_fast.test.ts @@ -1,5 +1,8 @@ import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; -import {prepareEpochProcessState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; +import { + createCachedValidatorsBeaconState, + prepareEpochProcessState, +} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; import {getAttestationDeltas} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/epoch"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; @@ -16,7 +19,8 @@ for (const testSuite of ["basic", "leak", "random"]) { const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - const process = prepareEpochProcessState(epochCtx, state); + const wrappedState = createCachedValidatorsBeaconState(state); + const process = prepareEpochProcessState(epochCtx, wrappedState); const [rewards, penalties] = getAttestationDeltas(epochCtx, process, state); return { rewards, diff --git a/packages/spec-test-runner/test/spec/sanity/blocks/sanity_blocks_fast.test.ts b/packages/spec-test-runner/test/spec/sanity/blocks/sanity_blocks_fast.test.ts index 8cc5d78d5112..6b39a6632a1d 100644 --- a/packages/spec-test-runner/test/spec/sanity/blocks/sanity_blocks_fast.test.ts +++ b/packages/spec-test-runner/test/spec/sanity/blocks/sanity_blocks_fast.test.ts @@ -1,22 +1,24 @@ import {join} from "path"; import {expect} from "chai"; import {BeaconState, SignedBeaconBlock} from "@chainsafe/lodestar-types"; -import {EpochContext, fastStateTransition, IStateContext} from "@chainsafe/lodestar-beacon-state-transition"; +import {EpochContext, fastStateTransition} from "@chainsafe/lodestar-beacon-state-transition"; import {config} from "@chainsafe/lodestar-config/mainnet"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IBlockSanityTestCase} from "./type"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util"; describeDirectorySpecTest( "block sanity mainnet", join(SPEC_TEST_LOCATION, "/tests/mainnet/phase0/sanity/blocks/pyspec_tests"), (testcase) => { const state = config.types.BeaconState.tree.createValue(testcase.pre); + const wrappedState = createCachedValidatorsBeaconState(state); const epochCtx = new EpochContext(config); - epochCtx.loadState(state); + epochCtx.loadState(wrappedState); + let stateContext = {epochCtx, state: wrappedState}; const verify = !!testcase.meta && !!testcase.meta.blsSetting && testcase.meta.blsSetting === BigInt(1); - let stateContext: IStateContext = {epochCtx, state}; for (let i = 0; i < Number(testcase.meta.blocksCount); i++) { stateContext = fastStateTransition(stateContext, testcase[`blocks_${i}`] as SignedBeaconBlock, { verifyStateRoot: verify, @@ -24,7 +26,7 @@ describeDirectorySpecTest( verifySignatures: verify, }); } - return stateContext.state; + return stateContext.state.getOriginalState(); }, { inputTypes: { diff --git a/packages/spec-test-runner/test/spec/sanity/slots/sanity_slots_fast.test.ts b/packages/spec-test-runner/test/spec/sanity/slots/sanity_slots_fast.test.ts index 644e1481b536..1447dff23108 100644 --- a/packages/spec-test-runner/test/spec/sanity/slots/sanity_slots_fast.test.ts +++ b/packages/spec-test-runner/test/spec/sanity/slots/sanity_slots_fast.test.ts @@ -4,6 +4,7 @@ import {config} from "@chainsafe/lodestar-config/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; import {processSlots} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/slot"; import {BeaconState} from "@chainsafe/lodestar-types"; +import {createCachedValidatorsBeaconState} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/util/interface"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessSlotsTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; @@ -15,7 +16,8 @@ describeDirectorySpecTest( const state = config.types.BeaconState.tree.createValue(testcase.pre); const epochCtx = new EpochContext(config); epochCtx.loadState(state); - processSlots(epochCtx, state, state.slot + Number(testcase.slots)); + const wrappedState = createCachedValidatorsBeaconState(state); + processSlots(epochCtx, wrappedState, state.slot + Number(testcase.slots)); return state; }, { diff --git a/yarn.lock b/yarn.lock index 55b751652a6d..4eef4f92e031 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5891,6 +5891,14 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= +fast-check@^1.15.1: + version "1.26.0" + resolved "https://registry.yarnpkg.com/fast-check/-/fast-check-1.26.0.tgz#3a85998a9c30ed7f58976276e06046645e0de18a" + integrity sha512-B1AjSfe0bmi6FdFIzmrrGSjrsF6e2MCmZiM6zJaRbBMP+gIvdNakle5FIMKi0xbS9KlN9BZho1R7oB/qoNIQuA== + dependencies: + pure-rand "^2.0.0" + tslib "^2.0.0" + fast-crc32c@^1.0.1: version "1.0.7" resolved "https://registry.yarnpkg.com/fast-crc32c/-/fast-crc32c-1.0.7.tgz#a7ab81b8665a6faee42ffe36ac930b15afc17396" @@ -10308,6 +10316,11 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +pure-rand@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-2.0.0.tgz#3324633545207907fe964c2f0ebf05d8e9a7f129" + integrity sha512-mk98aayyd00xbfHgE3uEmAUGzz3jCdm8Mkf5DUXUhc7egmOaGG2D7qhVlynGenNe9VaNJZvzO9hkc8myuTkDgw== + q@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -12057,6 +12070,11 @@ tslib@^1.8.1, tslib@^1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.11.1.tgz#eb15d128827fbee2841549e171f45ed338ac7e35" integrity sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA== +tslib@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" + integrity sha512-uZtkfKblCEQtZKBF6EBXVZeQNl82yqtDQdv+eck8u7tdPxjLu2/lp5/uPW+um2tpuxINHWy3GhiccY7QgEaVHQ== + tsutils@^3.17.1: version "3.17.1" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759" @@ -12853,4 +12871,3 @@ yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== -