From bbc819130ed599fda265d342cabe06115e6d8f51 Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 21:43:36 +0000 Subject: [PATCH 01/31] Report the model outcomes shown by consensus traces consensus-replay --outcomes names what each replayed model action does, as a branch of a handler in Model/Node.lean or an effect which handlers share, and reports the outcomes which a set of traces shows and those which none shows. The Lean workflow adds the report for the scenario traces to the summary of its replay job. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/README.md | 1 + .github/workflows/lean.yml | 8 + doc/architecture/verification.rst | 1 + lean/consensus/Consensus.lean | 1 + .../Consensus/Replay/Interpreter.lean | 14 +- lean/consensus/Consensus/Replay/Outcomes.lean | 594 ++++++++++++++++++ .../Consensus/Tests/ReplayOutcomes.lean | 148 +++++ lean/consensus/README.md | 1 + lean/consensus/replay/README.md | 26 +- lean/consensus/replay/ReplayMain.lean | 36 +- 10 files changed, 819 insertions(+), 11 deletions(-) create mode 100644 lean/consensus/Consensus/Replay/Outcomes.lean create mode 100644 lean/consensus/Consensus/Tests/ReplayOutcomes.lean diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e184db144f7..7ebc8d35a18 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -154,6 +154,7 @@ Caches are isolated by package and runner architecture and invalidated by change CCF package and common build outputs are not cached, and all verification steps still run. CCF consensus, KV, and consistency trace captures run in parallel with package checks. Three independent trace-validation jobs each build their own replayer, download the corresponding capture artifacts, and replay the traces without depending on the package matrix. +The CCF consensus replay job also adds to its summary the [outcomes](../../lean/consensus/replay/README.md#outcomes) of the model which the captured traces show, and those which none shows. Consistency capture reuses the existing `consistency_trace_validation` e2e test, and its replay job also checks the consistency replay fixtures, including election and rollback traces. A final summary job requires every package and trace job to succeed. diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index 339c19a9982..55696fd901f 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -272,6 +272,14 @@ jobs: echo "::endgroup::" done + - name: Report model outcomes shown by CCF consensus traces + shell: bash + run: | + set -euo pipefail + shopt -s failglob + lean/consensus/.lake/build/bin/consensus-replay --outcomes \ + "$RUNNER_TEMP"/cft-scenario-traces/*.ndjson >> "$GITHUB_STEP_SUMMARY" + models: name: Lean models and proofs needs: diff --git a/doc/architecture/verification.rst b/doc/architecture/verification.rst index 300e17fa852..cd6391ecbb3 100644 --- a/doc/architecture/verification.rst +++ b/doc/architecture/verification.rst @@ -81,6 +81,7 @@ The :ccf_repo:`Lean workflow <.github/workflows/lean.yml>` runs on pull requests * A package job for each of ``common``, ``consensus``, ``consistency``, ``disaster-recovery`` and ``kv`` runs the kernel and axiom check (``lake check``), import coverage and architecture checks, formatting and linting, and, where applicable, the package's replay fixtures. * Separate capture jobs build CCF and record consensus, KV and consistency traces from the tests described above. * Replay jobs build each replayer and validate every captured trace. + The consensus replay job also reports which outcomes of the model's handlers the captured traces show, and which none shows; see :ccf_repo:`lean/consensus/replay/README.md`. * A final ``Lean models and proofs`` job succeeds only if all of the above succeed. The disaster recovery traces are replayed by the SNP Genoa job in the main CI workflow instead, since that is where the recovery scenarios run. diff --git a/lean/consensus/Consensus.lean b/lean/consensus/Consensus.lean index 5d35ffae1b5..45a7341c9df 100644 --- a/lean/consensus/Consensus.lean +++ b/lean/consensus/Consensus.lean @@ -5,3 +5,4 @@ public import Consensus.Replay.Reduction import Consensus.Tests.CanonicalTraces import Consensus.Tests.ReplayDelivery +import Consensus.Tests.ReplayOutcomes diff --git a/lean/consensus/Consensus/Replay/Interpreter.lean b/lean/consensus/Consensus/Replay/Interpreter.lean index b9453554c73..d6d489a681e 100644 --- a/lean/consensus/Consensus/Replay/Interpreter.lean +++ b/lean/consensus/Consensus/Replay/Interpreter.lean @@ -4,6 +4,7 @@ module -- Licensed under the Apache 2.0 License. public import Consensus.Model +public import Consensus.Replay.Outcomes public import Lean.Data.Json public section @@ -62,6 +63,8 @@ structure ReplayState where state : Model.State /-- Envelopes queued by actions since the last send observation. -/ queued : List Envelope := [] + /-- Outcomes shown by the actions run so far, in order. -/ + shown : Array Outcome := #[] /-- A traced native send, or the proposal a nomination implies. -/ structure NativeSend where @@ -279,7 +282,11 @@ private def ReplayState.run (replay : ReplayState) (header : Header) | throw s!"disabled model action '{name}'" -- A step appends the acting node's sends to the network left after any delivery. let pending := replay.state.network.length - if action matches .deliver _ then 1 else 0 - return { state, queued := replay.queued ++ state.network.drop pending } + return { + state, + queued := replay.queued ++ state.network.drop pending, + shown := replay.shown ++ outcomes replay.state action + } /-- Observations never change model state or repair a disabled action. -/ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instruction) @@ -342,6 +349,8 @@ structure Result where instructions : Nat actions : Nat observations : Nat + /-- Outcomes shown by the actions, in order. -/ + outcomes : Array Outcome deriving Repr, BEq def replay (header : Header) (instructions : Array Instruction) @@ -369,7 +378,8 @@ def replay (header : Header) (instructions : Array Instruction) return { instructions := instructions.size, actions, - observations := instructions.size - actions + observations := instructions.size - actions, + outcomes := state.shown } end Consensus.Replay diff --git a/lean/consensus/Consensus/Replay/Outcomes.lean b/lean/consensus/Consensus/Replay/Outcomes.lean new file mode 100644 index 00000000000..16c4dc42a4e --- /dev/null +++ b/lean/consensus/Consensus/Replay/Outcomes.lean @@ -0,0 +1,594 @@ +module + +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. + +public import Consensus.Model + +public section + +/-! +# Outcomes of model steps + +An outcome names one thing which a node's step does: a branch of a handler in +`Model/Node.lean`, or an effect which several handlers share, such as a commit. +Replay collects the outcomes of the actions it runs, so that what the traces of the +implementation exercise can be compared with what the model allows. +A step may show several outcomes. +-/ + +namespace Consensus.Replay + +open Model.Node +open CCF (Finset) +open CCF.MultiNodeTransitionSystem (nodeState) + +inductive Outcome where + -- `appendBatch` + /-- A transaction which changes neither membership nor retirement is appended. -/ + | appendTransaction + | appendSignature + | appendConfiguration + /-- The peers which a configuration adds are sent a heartbeat as it is appended. -/ + | appendConfigurationToNewPeers + | appendRetirementConfirmation + | appendSeveralEntries + /-- A batch sends entries before its commit check. -/ + | appendFlushes + -- `sendEntries` + | sendEntries + | sendHeartbeat + /-- The destination has been removed, and its retirement is not confirmed yet. -/ + | sendToRemovedPeer + -- `startElection` + | electionStarted + /-- A node which is already campaigning times out again. -/ + | electionRetried + /-- The node is a quorum by itself, and becomes primary at once. -/ + | electionWonWithoutPeers + /-- The node is in no signed active configuration, and campaigns as a removed node. -/ + | electionByRemovedNode + -- `checkQuorum`, `nominateSuccessor` and `initAsBackup` + | checkQuorumStepsDown + | nominationSent + | nominationWithoutSuccessor + | joinEmpty + | joinFromSnapshot + -- `handleAppendEntriesRequest` + /-- Rejected: the request is from an earlier term. -/ + | appendEntriesStaleTerm + /-- Rejected: the log ends before the previous index. -/ + | appendEntriesMissingPrevious + /-- Rejected with a hint: the entry at the previous index has another term. -/ + | appendEntriesMismatchedPrevious + /-- Ignored: the previous index is below the commit index. -/ + | appendEntriesBelowCommit + /-- Ignored: an entry conflicts, and the node may not roll back. -/ + | appendEntriesConflictIgnored + /-- A conflicting suffix is rolled back. -/ + | appendEntriesTruncates + | appendEntriesAppends + /-- Accepted: every entry is already in the log. -/ + | appendEntriesDuplicate + | appendEntriesHeartbeat + /-- A candidate learns of the primary of its own term. -/ + | appendEntriesDemotesCandidate + /-- The commit index of the follower advances. -/ + | appendEntriesCommits + -- `handleAppendEntriesResponse` + | appendEntriesResponseFromUnknownPeer + | appendEntriesResponseToNonPrimary + | ackAdvancesMatch + /-- An acknowledgement at the match index, as for a heartbeat. -/ + | ackRepeated + /-- An acknowledgement below the match index, which arrives late. -/ + | ackBelowMatch + /-- Ignored: an acknowledgement from an earlier term. -/ + | ackFromOtherTerm + /-- A rejection moves the send index back. -/ + | nackRewinds + | nackWithoutRewind + -- `handleRequestVoteRequest` + | voteGranted + | voteRefusedStaleTerm + /-- Refused only because the log of the candidate is behind. -/ + | voteRefusedLog + /-- Refused only because the voter knows a primary in this term. -/ + | voteRefusedPrimaryKnown + /-- Refused only because the voter voted for another candidate. -/ + | voteRefusedAlreadyVoted + | voteRefusedSeveralReasons + -- `handleRequestPreVoteRequest` + | preVoteGranted + | preVoteRefusedStaleTerm + | preVoteRefusedLog + -- `handleRequestVoteResponse` + | voteResponseFromUnknownPeer + | voteResponseFromOtherTerm + /-- Ignored: the node gave up its candidacy, and campaigns again in the same term. -/ + | voteResponseToPreVoteCandidate + /-- Ignored: the node is no longer campaigning. -/ + | voteResponseToNonCandidate + | voteResponseRefusal + | voteResponseCounted + | voteResponseElects + -- `handleRequestPreVoteResponse` + | preVoteResponseFromUnknownPeer + | preVoteResponseFromOtherTerm + /-- Ignored: the node is a candidate already, and a pre-vote is not a vote. -/ + | preVoteResponseToCandidate + /-- Ignored: the node is no longer campaigning. -/ + | preVoteResponseToNonCandidate + | preVoteResponseRefusal + | preVoteResponseCounted + | preVoteResponseStartsElection + -- `handleProposeRequestVoteRequest` + | proposalAccepted + | proposalFromOtherTerm + /-- Ignored: the node may not start an election. -/ + | proposalIneligible + -- `observeTerm` + | higherTermAdopted + | higherTermDeposesPrimary + -- `becomePrimary` + /-- The new primary drops the entries after its last signature. -/ + | electedTruncatesSuffix + /-- The election needed a quorum of several active configurations. -/ + | electedWithJointQuorum + -- `advanceCommit` + | primaryCommits + /-- The commit needed a quorum of several active configurations. -/ + | commitWithJointQuorum + /-- The commit stops before a pending configuration. -/ + | commitBelowPendingConfiguration + /-- A quorum holds a signature of an earlier term, which stays uncommitted. -/ + | commitDeferredForOldTerm + -- `NodeState.retirement` + | retirementOrdered + | retirementSigned + | retirementCompleted + | retirementCommitted + /-- A primary nominates a successor as its retirement is confirmed. -/ + | retirementCommitNominates + /-- The signature after the removal of the node is rolled back, but not the removal. -/ + | retirementSignatureRolledBack + /-- The removal of the node is rolled back. -/ + | retirementRolledBack +deriving DecidableEq, Repr + +namespace Outcome + +/-- Every outcome, in the order of declaration. -/ +def all : List Outcome := + [ + .appendTransaction, + .appendSignature, + .appendConfiguration, + .appendConfigurationToNewPeers, + .appendRetirementConfirmation, + .appendSeveralEntries, + .appendFlushes, + .sendEntries, + .sendHeartbeat, + .sendToRemovedPeer, + .electionStarted, + .electionRetried, + .electionWonWithoutPeers, + .electionByRemovedNode, + .checkQuorumStepsDown, + .nominationSent, + .nominationWithoutSuccessor, + .joinEmpty, + .joinFromSnapshot, + .appendEntriesStaleTerm, + .appendEntriesMissingPrevious, + .appendEntriesMismatchedPrevious, + .appendEntriesBelowCommit, + .appendEntriesConflictIgnored, + .appendEntriesTruncates, + .appendEntriesAppends, + .appendEntriesDuplicate, + .appendEntriesHeartbeat, + .appendEntriesDemotesCandidate, + .appendEntriesCommits, + .appendEntriesResponseFromUnknownPeer, + .appendEntriesResponseToNonPrimary, + .ackAdvancesMatch, + .ackRepeated, + .ackBelowMatch, + .ackFromOtherTerm, + .nackRewinds, + .nackWithoutRewind, + .voteGranted, + .voteRefusedStaleTerm, + .voteRefusedLog, + .voteRefusedPrimaryKnown, + .voteRefusedAlreadyVoted, + .voteRefusedSeveralReasons, + .preVoteGranted, + .preVoteRefusedStaleTerm, + .preVoteRefusedLog, + .voteResponseFromUnknownPeer, + .voteResponseFromOtherTerm, + .voteResponseToPreVoteCandidate, + .voteResponseToNonCandidate, + .voteResponseRefusal, + .voteResponseCounted, + .voteResponseElects, + .preVoteResponseFromUnknownPeer, + .preVoteResponseFromOtherTerm, + .preVoteResponseToCandidate, + .preVoteResponseToNonCandidate, + .preVoteResponseRefusal, + .preVoteResponseCounted, + .preVoteResponseStartsElection, + .proposalAccepted, + .proposalFromOtherTerm, + .proposalIneligible, + .higherTermAdopted, + .higherTermDeposesPrimary, + .electedTruncatesSuffix, + .electedWithJointQuorum, + .primaryCommits, + .commitWithJointQuorum, + .commitBelowPendingConfiguration, + .commitDeferredForOldTerm, + .retirementOrdered, + .retirementSigned, + .retirementCompleted, + .retirementCommitted, + .retirementCommitNominates, + .retirementSignatureRolledBack, + .retirementRolledBack + ] + +/-- A report lists `all`, so it must not omit an outcome. -/ +theorem mem_all (outcome : Outcome) : outcome ∈ all := by + cases outcome <;> decide + +/-- The name of the constructor. -/ +def name (outcome : Outcome) : String := + ((reprStr outcome).splitOn ".").getLastD "" + +end Outcome + +private abbrev LocalStep := + CCF.MultiNodeTransitionSystem.LocalStep NodeID NodeState Event Message Notification + +/-- The step which the acting node takes for `action`, unless it disables the action. -/ +private def localStep (state : Model.State) (action : Model.Action) + : Option LocalStep := do + let (node, event) := + match action with + | .local node input => (node, Event.internal input) + | .deliver envelope => + (envelope.target, Event.receive envelope.source envelope.payload) + let before ← nodeState state node + let (after, effects) ← (step (CCF.Capabilities.record node) node before event).run {} + return { node, before, action := event, after, effects } + +private def tag (condition : Prop) [Decidable condition] (outcome : Outcome) + : List Outcome := + if condition then [outcome] else [] + +private def either (condition : Prop) [Decidable condition] (yes no : Outcome) + : List Outcome := + [if condition then yes else no] + +private def sent (s : LocalStep) : List Message := + s.effects.outgoing.map (·.payload) + +/-! ## Local inputs -/ + +private def appendOutcomes (s : LocalStep) (entries : List AppendEntryInput) + : List Outcome := + let any (property : EntryContent → Bool) := + entries.any fun entry => property entry.content + let plain (content : EntryContent) : Bool := + !content.isSignature && content.metadata == {} + let confirms (content : EntryContent) : Bool := + decide content.metadata.retirementConfirmations.Nonempty + let peers := eligiblePeers s.before s.node + let toNewPeer (envelope : Model.Envelope) : Bool := + envelope.payload matches .appendEntriesRequest _ && decide (envelope.target ∉ peers) + tag (any plain) .appendTransaction + ++ tag (any (·.isSignature)) .appendSignature + ++ tag (any (·.metadata.configuration.isSome)) .appendConfiguration + ++ tag (s.effects.outgoing.any toNewPeer) .appendConfigurationToNewPeers + ++ tag (any confirms) .appendRetirementConfirmation + ++ tag (entries.length > 1) .appendSeveralEntries + ++ tag (entries.any (!·.flushes.isEmpty)) .appendFlushes + +private def sendOutcomes (s : LocalStep) (destination endIndex : Nat) : List Outcome := + let entries := messageEntries s.before.log (s.before.sentIndex destination) endIndex + either entries.isEmpty .sendHeartbeat .sendEntries + ++ tag (destination ∉ s.before.activeMembers) .sendToRemovedPeer + +private def electionOutcomes (s : LocalStep) : List Outcome := + either (s.before.role = .follower) .electionStarted .electionRetried + ++ tag (s.after.role = .primary) .electionWonWithoutPeers + ++ tag (s.node ∉ configurationMembers s.before.signedActiveConfigurations) + .electionByRemovedNode + +/-! ## Handling AppendEntries -/ + +/-- `state` is the receiver after `observeTerm`, wherever a handler takes it. -/ +private def appendEntriesRequestOutcomes (s : LocalStep) (state : NodeState) + (request : AppendEntriesRequest) + : List Outcome := + let response := + (sent s).findSome? + fun + | .appendEntriesResponse response => some response + | _ => none + let result : Outcome := + match response with + | none => + if request.prevLogIndex < state.commitIndex then + .appendEntriesBelowCommit + else + .appendEntriesConflictIgnored + | some response => + if !response.success then + if request.term < state.currentTerm then + .appendEntriesStaleTerm + else if termAt state.log request.prevLogIndex = 0 then + .appendEntriesMissingPrevious + else + .appendEntriesMismatchedPrevious + else if !state.log.isPrefixOf s.after.log then + .appendEntriesTruncates + else if state.log.length < s.after.log.length then + .appendEntriesAppends + else if request.entries.isEmpty then + .appendEntriesHeartbeat + else + .appendEntriesDuplicate + [result] + ++ tag + (request.term = state.currentTerm + /\ (state.role = .candidate \/ state.role = .preVoteCandidate)) + .appendEntriesDemotesCandidate + ++ tag (s.before.commitIndex < s.after.commitIndex) .appendEntriesCommits + +/-- +Whether a quorum holds a signature of an earlier term above the commit index, which +`highestCommittableIndex` does not commit. +-/ +private def deferredForOldTerm (self : NodeID) (state : NodeState) : Bool := + let replicatedBy index := + state.activeMembers.filter fun node => node = self \/ state.matchIndex node >= index + (List.range (state.log.length + 1)).any + fun index => + decide + (index > state.commitIndex + /\ isSignatureAt state.log index = true + /\ termAt state.log index ≠ state.currentTerm + /\ hasQuorum + (state.activeConfigurations.filter + fun configuration => configuration.index <= index) + (replicatedBy index)) + +private def appendEntriesResponseOutcomes (s : LocalStep) (state : NodeState) + (source : NodeID) (response : AppendEntriesResponse) + : List Outcome := + let rewinds := decide (s.after.sentIndex source < state.sentIndex source) + let matched := state.matchIndex source + if state.role ≠ .primary then + [.appendEntriesResponseToNonPrimary] + else if !response.success then + either rewinds .nackRewinds .nackWithoutRewind + else if response.term ≠ state.currentTerm then + [.ackFromOtherTerm] + else if response.lastLogIndex < matched then + [.ackBelowMatch] + else if response.lastLogIndex = matched then + [.ackRepeated] + else + .ackAdvancesMatch :: tag (deferredForOldTerm s.node s.after) .commitDeferredForOldTerm + +/-! ## Handling elections -/ + +private def voteRequestOutcomes (s : LocalStep) (state : NodeState) (source : NodeID) + (request : RequestVoteRequest) + : List Outcome := + let granted := + (sent s).any + fun + | .requestVoteResponse response => response.voteGranted + | _ => false + if granted then + [.voteGranted] + else if request.term ≠ state.currentTerm then + [.voteRefusedStaleTerm] + else + let reasons := + tag (Not (voteLogUpToDate state request)) .voteRefusedLog + ++ tag state.primaryKnown .voteRefusedPrimaryKnown + ++ tag (Not (state.votedFor = none \/ state.votedFor = some source)) + .voteRefusedAlreadyVoted + match reasons with + | [reason] => [reason] + | _ => [.voteRefusedSeveralReasons] + +private def preVoteRequestOutcomes (s : LocalStep) (state : NodeState) + (request : RequestVoteRequest) + : List Outcome := + let granted := + (sent s).any + fun + | .requestPreVoteResponse response => response.voteGranted + | _ => false + if granted then + [.preVoteGranted] + else if request.term ≠ state.currentTerm then + [.preVoteRefusedStaleTerm] + else + [.preVoteRefusedLog] + +/-- +A response with the term of the node leaves `observeTerm` without effect, so the +state before the step decides. +-/ +private def voteResponseOutcomes (s : LocalStep) (response : RequestVoteResponse) + : List Outcome := + if response.term ≠ s.before.currentTerm then + [.voteResponseFromOtherTerm] + else if s.before.role = .preVoteCandidate then + [.voteResponseToPreVoteCandidate] + else if s.before.role ≠ .candidate then + [.voteResponseToNonCandidate] + else if !response.voteGranted then + [.voteResponseRefusal] + else if s.after.role = .primary then + [.voteResponseElects] + else + [.voteResponseCounted] + +private def preVoteResponseOutcomes (s : LocalStep) (response : RequestVoteResponse) + : List Outcome := + if response.term ≠ s.before.currentTerm then + [.preVoteResponseFromOtherTerm] + else if s.before.role = .candidate then + [.preVoteResponseToCandidate] + else if s.before.role ≠ .preVoteCandidate then + [.preVoteResponseToNonCandidate] + else if !response.voteGranted then + [.preVoteResponseRefusal] + else if s.after.role = .preVoteCandidate then + [.preVoteResponseCounted] + else + [.preVoteResponseStartsElection] + +/-- An accepted proposal starts an election, which increments the term. -/ +private def proposalOutcomes (s : LocalStep) (term : Nat) : List Outcome := + if term ≠ s.before.currentTerm then + [.proposalFromOtherTerm] + else if s.after.currentTerm = s.before.currentTerm then + [.proposalIneligible] + else + [.proposalAccepted] + +private def receiveOutcomes (s : LocalStep) (source : NodeID) (message : Message) + : List Outcome := + let unknownPeer : Option Outcome := + if source ∈ eligiblePeers s.before s.node then + none + else + match message with + | .appendEntriesResponse _ => some .appendEntriesResponseFromUnknownPeer + | .requestVoteResponse _ => some .voteResponseFromUnknownPeer + | .requestPreVoteResponse _ => some .preVoteResponseFromUnknownPeer + | _ => none + if let some outcome := unknownPeer then + [outcome] + else + let state := observeTerm s.before message + let handled := + match message with + | .appendEntriesRequest request => appendEntriesRequestOutcomes s state request + | .appendEntriesResponse response => + appendEntriesResponseOutcomes s state source response + | .requestVoteRequest request => voteRequestOutcomes s state source request + | .requestVoteResponse response => voteResponseOutcomes s response + | .requestPreVoteRequest request => preVoteRequestOutcomes s state request + | .requestPreVoteResponse response => preVoteResponseOutcomes s response + | .proposeRequestVoteRequest term => proposalOutcomes s term + let higherTerm := s.before.currentTerm < state.currentTerm + handled + ++ tag (higherTerm /\ s.before.role ≠ .primary) .higherTermAdopted + ++ tag (higherTerm /\ s.before.role = .primary) .higherTermDeposesPrimary + +/-! ## Effects shared by handlers -/ + +private def electedOutcomes (s : LocalStep) : List Outcome := + if s.before.role ≠ .primary /\ s.after.role = .primary then + tag (s.after.log.length < s.before.log.length) .electedTruncatesSuffix + ++ tag (s.before.activeConfigurations.length > 1) .electedWithJointQuorum + else + [] + +/-- +A follower commits only in `handleAppendEntriesRequest`, and a joiner starts committed. +As in `highestCommittableIndex`, a commit needs a quorum of each configuration which is +active at the previous commit index and at or below the new one. +-/ +private def commitOutcomes (s : LocalStep) : List Outcome := + let asPrimary : Bool := + decide (s.before.role = .primary \/ s.after.role = .primary) + && !(s.action matches .receive _ (.appendEntriesRequest _)) + if asPrimary /\ s.before.commitIndex < s.after.commitIndex then + let active := activeConfigurations s.after.log s.before.commitIndex + let reached := + active.filter fun configuration => configuration.index <= s.after.commitIndex + [.primaryCommits] + ++ tag (reached.length > 1) .commitWithJointQuorum + ++ tag (reached.length < active.length) .commitBelowPendingConfiguration + else + [] + +private def retirementOutcomes (s : LocalStep) : List Outcome := + let before := (s.before.retirement s.node).phase + let after := (s.after.retirement s.node).phase + if before = after then + [] + else + match after with + | .noRetirementObserved => [.retirementRolledBack] + | .retirementOrdered => + either (before = .retirementSigned) .retirementSignatureRolledBack + .retirementOrdered + | .retirementSigned => [.retirementSigned] + | .retirementCompleted => [.retirementCompleted] + | .retiredCommitted => + [.retirementCommitted] + ++ tag + (s.before.role = .primary + /\ (sent s).any (· matches .proposeRequestVoteRequest _)) + .retirementCommitNominates + +/-! ## The step -/ + +/-- What the step of the acting node shows; nothing if the node disables `action`. -/ +def outcomes (state : Model.State) (action : Model.Action) : List Outcome := + match localStep state action with + | none => [] + | some s => + let handled := + match s.action with + | .internal (.append entries) => appendOutcomes s entries + | .internal (.sendEntries destination endIndex) => + sendOutcomes s destination endIndex + | .internal (.timeout .election) => electionOutcomes s + | .internal (.timeout .checkQuorum) => [.checkQuorumStepsDown] + | .internal .nominateSuccessor => + either s.effects.outgoing.isEmpty .nominationWithoutSuccessor .nominationSent + | .internal (.initAsBackup entries) => + either entries.isEmpty .joinEmpty .joinFromSnapshot + | .receive source message => receiveOutcomes s source message + handled ++ electedOutcomes s ++ commitOutcomes s ++ retirementOutcomes s + +/-- +A Markdown table of the steps and traces which show each outcome, with the first such +trace, followed by the outcomes which no trace shows. +-/ +def report (traces : Array (String × Array Outcome)) : String := + let rows := + Outcome.all.map + fun outcome => + let showing := traces.filter fun (_, shown) => shown.contains outcome + let steps := traces.foldl (fun total (_, shown) => total + shown.count outcome) 0 + (outcome.name, steps, showing.size, (showing[0]?.map Prod.fst).getD "") + let missing := (rows.filter fun (_, steps, _) => steps == 0).map (·.1) + let lines := + ["| Outcome | Steps | Traces | First trace |", "| --- | ---: | ---: | --- |"] + ++ rows.map + (fun (name, steps, count, first) => s!"| {name} | {steps} | {count} | {first} |") + ++ [ + "", + s!"Traces: {traces.size}. Outcomes shown: {rows.length - missing.length} of {rows.length}." + ] + ++ if missing.isEmpty then [] else ["", s!"Not shown: {", ".intercalate missing}."] + "\n".intercalate lines + +end Consensus.Replay diff --git a/lean/consensus/Consensus/Tests/ReplayOutcomes.lean b/lean/consensus/Consensus/Tests/ReplayOutcomes.lean new file mode 100644 index 00000000000..d8651abdd7c --- /dev/null +++ b/lean/consensus/Consensus/Tests/ReplayOutcomes.lean @@ -0,0 +1,148 @@ +module + +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. + +meta import Consensus.Replay.Outcomes + +public import Consensus.Replay.Outcomes + +public section + +namespace Consensus.Tests.ReplayOutcomes + +open Consensus Consensus.Replay Consensus.Model.Node +open CCF (Finset) + +private def configuration : Finset NodeID := {0, 1, 2} + +/-- Node 0 is the primary of term 2, holding the configuration; the others are followers. -/ +private def node (id : NodeID) : NodeState := Model.initialNodeState configuration id + +private def transaction (term : Nat) : Entry := { term, content := .transaction 1 {} } + +private def expect (name : String) (actual expected : List Outcome) : IO Unit := do + unless actual == expected do + throw + (IO.userError + s!"{name}: shown {actual.map Outcome.name}, expected {expected.map Outcome.name}") + +/-- What `target`, in `state`, shows on receiving `payload` from `source`. -/ +private def receive (target : NodeID) (state : NodeState) (source : NodeID) + (payload : Message) + : List Outcome := + let envelope : Model.Envelope := { source, target, payload } + outcomes { nodes := [(target, state)], network := [envelope] } (.deliver envelope) + +/-- Deliver the oldest pending message along this directed link. -/ +private def oldest (source target : NodeID) (state : Model.State) : Option Model.Action := + (state.network.find? + fun envelope => envelope.source == source && envelope.target == target).map + fun envelope => .deliver envelope + +/-- A batch is replicated to one follower, whose acknowledgement commits it. -/ +private def replication : IO Unit := do + let steps : List (String × (Model.State → Option Model.Action) × List Outcome) := + [ + ( + "append", + fun _ => + some + (.local 0 + (.append [{ content := .transaction 10 {} }, { content := .signature }])), + [.appendTransaction, .appendSignature, .appendSeveralEntries] + ), + ("send", fun _ => some (.local 0 (.sendEntries 1 3)), [.sendEntries]), + ("receive", oldest 0 1, [.appendEntriesAppends]), + ("acknowledge", oldest 1 0, [.ackAdvancesMatch, .primaryCommits]) + ] + let mut state : Model.State := { nodes := [0, 1, 2].map fun id => (id, node id) } + for (name, action, expected) in steps do + let some action := action state + | throw (IO.userError s!"{name}: no pending message") + expect name (outcomes state action) expected + let some next := Model.networkStep configuration state action + | throw (IO.userError s!"{name}: disabled action") + state := next + +/-- A refusal names its reason only when it is the only one. -/ +private def refusals : IO Unit := do + let request : Message := + .requestVoteRequest { term := 3, lastCommittableTerm := 0, lastCommittableIndex := 0 } + let voter := { node 1 with currentTerm := 3, primaryKnown := true } + expect "primary known" (receive 1 voter 2 request) [.voteRefusedPrimaryKnown] + expect "primary known and voted" (receive 1 { voter with votedFor := some 0 } 2 request) + [.voteRefusedSeveralReasons] + +/-- The requests which a follower ignores without a response are told apart. -/ +private def conflicts : IO Unit := do + let follower := { node 1 with currentTerm := 3, log := [transaction 2, transaction 2] } + let request : Message := + .appendEntriesRequest + { + term := 3 + prevLogIndex := 1 + prevLogTerm := 2 + entries := [transaction 3] + primaryCommit := 0 + } + expect "conflict" (receive 1 follower 0 request) [.appendEntriesConflictIgnored] + expect "conflict for a new follower" + (receive 1 { follower with isNewFollower := true } 0 request) + [.appendEntriesTruncates] + expect "below commit" (receive 1 { follower with commitIndex := 2 } 0 request) + [.appendEntriesBelowCommit] + +/-- A quorum acknowledges a signature of term 2, which a primary of term 3 does not commit. -/ +private def deferredCommit : IO Unit := do + let primary := + { + node 0 with + currentTerm := 3 + log := (node 0).log ++ [{ term := 2, content := .signature }] + } + let acknowledgement : Message := + .appendEntriesResponse { term := 3, success := true, lastLogIndex := 2 } + expect "signature of an earlier term" (receive 0 primary 1 acknowledgement) + [.ackAdvancesMatch, .commitDeferredForOldTerm] + +/-- Responses which arrive after the node has moved on are told apart from refusals. -/ +private def lateResponses : IO Unit := do + let candidate := + { node 1 with role := .candidate, currentTerm := 3, log := (node 0).log } + let preVote : Message := .requestPreVoteResponse { term := 3, voteGranted := true } + expect "pre-vote for a candidate" (receive 1 candidate 2 preVote) + [.preVoteResponseToCandidate] + let acknowledgement : Message := + .appendEntriesResponse { term := 2, success := true, lastLogIndex := 0 } + expect "acknowledgement below the match index" + (receive 0 { node 0 with matchIndex := fun _ => 1 } 1 acknowledgement) + [.ackBelowMatch] + +private def reporting : IO Unit := do + let traces : Array (String × Array Outcome) := + #[("first", #[.voteGranted, .voteGranted]), ("second", #[.voteGranted])] + let lines := (report traces).splitOn "\n" + let expected := + [ + "| voteGranted | 3 | 2 | first |", + "| joinEmpty | 0 | 0 | |", + s!"Traces: 2. Outcomes shown: 1 of {Outcome.all.length}." + ] + for line in expected do + unless lines.contains line do + throw (IO.userError s!"report: missing line '{line}'") + +/-- Checks the outcomes of chosen steps, and the report of several traces. -/ +def run : IO Unit := do + replication + refusals + conflicts + deferredCommit + lateResponses + reporting + IO.println "Replay outcome checks passed." + +#eval run + +end Consensus.Tests.ReplayOutcomes diff --git a/lean/consensus/README.md b/lean/consensus/README.md index 66fdae65fa8..ecf5fa25f1e 100644 --- a/lean/consensus/README.md +++ b/lean/consensus/README.md @@ -72,4 +72,5 @@ To list the safety-invariant declarations used by the exported theorems, run `la The [trace replayer](replay/README.md) checks finite driver observations against direct model transitions, independently of proofs. It checks each native send against the model, delivers any matching queued packet, treats undelivered packets as lost, and never repairs model state. +It also reports which [outcomes](replay/README.md#outcomes) of the model's handlers a set of traces shows, and which it never shows. The current native Fluentd capture path and replay's assurance limits are documented there. diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index e39ffb57fd3..5036bd246ba 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -1,7 +1,7 @@ # CCF consensus protocol trace replay Replay native `cft_driver` captures against the Lean model of the CCF consensus protocol. -The existing `cft_scenario_test` collects native traces; `consensus-replay` validates one trace per invocation. +The existing `cft_scenario_test` collects native traces; `consensus-replay` validates one trace per invocation, or reports the [outcomes](#outcomes) of several. ## Run replay @@ -32,6 +32,26 @@ done The scenario runner writes the raw records to `consensus/raw/`. The Fluentd tag is `ccf.cft`; packet message names are unprefixed, for example `append_entries` and `request_vote`. +## Outcomes + +Replay names what each model action does: the branch which a handler in [`Node.lean`](../Consensus/Model/Node.lean) takes, or an effect which handlers share, such as a commit. +[`Outcomes.lean`](../Consensus/Replay/Outcomes.lean) declares these outcomes in the order of the handlers. +To report how many steps and traces show each outcome, and which outcomes no trace shows, validate the traces together: + +```sh +./.lake/build/bin/consensus-replay --outcomes /tmp/cft-build/consensus/raw/*.ndjson +``` + +An outcome which no trace shows is a missing scenario, a step which the driver or the reduction cannot produce, or behaviour which the model allows and the implementation lacks. +For example, a nomination without a successor traces no `step_down_and_nominate_successor`, so replay runs no model action for it. + +To compare with the implementation, configure the scenario build with `-DCOVERAGE=ON` and run the scenario test. +[`coverage.sh`](../../../scripts/coverage.sh) then lists the lines of [`cft.h`](../../../src/consensus/aft/cft.h) which the scenarios do not execute: + +```sh +(cd /tmp/cft-build && "$OLDPWD/../../scripts/coverage.sh" --show-uncovered cft_driver) +``` + ## Replay contract Every protocol event must match a named rule; bootstrap selects the initial singleton membership. @@ -52,8 +72,10 @@ Native code does not trace the `propose_request_vote` send that follows `step_do - [`Input.lean`](../Consensus/Replay/Input.lean): decode events, carry optional command provenance and collect identities. - [`Reduction.lean`](../Consensus/Replay/Reduction.lean): named rules, explicit boundaries and rule-tagged snapshot checks. - [`Interpreter.lean`](../Consensus/Replay/Interpreter.lean): boundary-state field comparisons, packet selection, send checks and model execution. -- [`ReplayMain.lean`](ReplayMain.lean): single-trace validation CLI. +- [`Outcomes.lean`](../Consensus/Replay/Outcomes.lean): the outcomes of model steps, and their report. +- [`ReplayMain.lean`](ReplayMain.lean): CLI which validates one trace, or reports the outcomes of several. - [`ReplayDelivery.lean`](../Consensus/Tests/ReplayDelivery.lean): delivery selection among equal rendered packets. +- [`ReplayOutcomes.lean`](../Consensus/Tests/ReplayOutcomes.lean): outcomes of chosen steps, and the report of several traces. - [`cft_scenarios_runner.py`](../../../tests/cft_scenarios_runner.py): existing native scenario runner and capture generation. See [failure fixtures](fixtures/README.md) for examples and expected diagnostics. diff --git a/lean/consensus/replay/ReplayMain.lean b/lean/consensus/replay/ReplayMain.lean index dd7f7c2b12c..e77be24eb9c 100644 --- a/lean/consensus/replay/ReplayMain.lean +++ b/lean/consensus/replay/ReplayMain.lean @@ -9,16 +9,38 @@ public section open Lean Consensus.Replay +/-- Reduce one trace, and replay it against the model. -/ +private def validate (path : String) : IO (Except String Result) := do + let records ← readLog path + return do + let (events, declared) ← prepareEvents (← records) + let reduced ← reduce events declared + replay reduced.header reduced.instructions + +/-- Validate each trace, then report the outcomes which they show together. -/ +private def reportOutcomes (paths : List String) : IO UInt32 := do + let mut traces := #[] + for path in paths do + match ← validate path with + | .error error => IO.eprintln s!"{path}: {error}"; return 1 + | .ok result => + let name := (System.FilePath.mk path).fileStem.getD path + traces := traces.push (name, result.outcomes) + IO.println (report traces) + return 0 + +private def usage : IO UInt32 := do + IO.eprintln "usage: consensus-replay TRACE.ndjson" + IO.eprintln " consensus-replay --outcomes TRACE.ndjson..." + return 2 + def main (args : List String) : IO UInt32 := do try match args with + | ["--outcomes"] => usage + | "--outcomes" :: paths => reportOutcomes paths | [path] => - let records ← readLog path - let result := do - let (events, declared) ← prepareEvents (← records) - let reduced ← reduce events declared - replay reduced.header reduced.instructions - match result with + match ← validate path with | .error error => IO.eprintln s!"{path}: {error}"; return 1 | .ok result => let result := @@ -31,7 +53,7 @@ def main (args : List String) : IO UInt32 := do ] IO.println result.compress return 0 - | _ => IO.eprintln "usage: consensus-replay TRACE.ndjson"; return 2 + | _ => usage catch error => IO.eprintln s!"{error}" return 1 From 486080d6504869a174abbc8ee83e273aebc9740f Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 22:14:00 +0000 Subject: [PATCH 02/31] Let consensus scenarios deliver any pending message dispatchNth and dropNth deliver or drop the message at a position among those undelivered on a link, where dispatchSingle delivers the oldest. pendingMessages lists them, as the packets of a trace. The reordered_messages scenario uses them to reach two cases which no scenario showed: a follower ignores entries below its commit index, and a primary ignores an acknowledgement below its match index. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.cpp | 87 +++++++++++-- src/consensus/aft/test/driver.h | 114 ++++++++++++++---- tests/cft_scenarios/CftDriver.lean | 18 +++ tests/cft_scenarios/Scenarios.lean | 1 + .../Scenarios/reordered_messages.lean | 94 +++++++++++++++ 5 files changed, 277 insertions(+), 37 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/reordered_messages.lean diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index a559c7c2dc9..92fc60e54c3 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace @@ -65,14 +66,22 @@ namespace } // Lean cannot unwind C++ exceptions, so return them as IO errors, which fail - // the scenario + // the scenario. f returns nothing, or the Lean value which the IO action + // returns. template lean_obj_res io(F&& f) { try { - f(); - return lean_io_result_mk_ok(lean_box(0)); + if constexpr (std::is_void_v>) + { + f(); + return lean_io_result_mk_ok(lean_box(0)); + } + else + { + return lean_io_result_mk_ok(f()); + } } catch (const std::exception& e) { @@ -81,6 +90,23 @@ namespace } } + // Records a step in the trace, and names it if it fails + template + auto traced(const std::string& cmd, F&& f) + { + ++step_count; + try + { + ccf::tracing::emit(aft::trace::cft_trace_tag, "cmd", cmd); + return f(); + } + catch (const std::exception& e) + { + throw std::runtime_error( + fmt::format("Step {} ({}) failed: {}", step_count, cmd, e.what())); + } + } + enum class Invariants : uint8_t { CHECK, @@ -93,23 +119,26 @@ namespace const std::string& cmd, F&& f, Invariants invariants = Invariants::CHECK) { return io([&]() { - ++step_count; - try - { - ccf::tracing::emit(aft::trace::cft_trace_tag, "cmd", cmd); + traced(cmd, [&]() { f(); if (invariants == Invariants::CHECK) { driver->assert_invariants(); } - } - catch (const std::exception& e) - { - throw std::runtime_error( - fmt::format("Step {} ({}) failed: {}", step_count, cmd, e.what())); - } + }); }); } + + lean_obj_res strings(const std::vector& items) + { + auto* array = lean_mk_empty_array(); + for (const auto& item : items) + { + array = lean_array_push( + array, lean_mk_string_from_bytes(item.data(), item.size())); + } + return array; + } } extern "C" @@ -343,6 +372,28 @@ extern "C" }); } + lean_obj_res cft_dispatch_nth( + b_lean_obj_arg source, b_lean_obj_arg target, uint64_t position) + { + const auto src = text(source); + const auto dst = text(target); + return step(command("dispatch_nth", src, dst, position), [&]() { + driver->dispatch_nth(src, dst, position); + }); + } + + lean_obj_res cft_pending_messages( + b_lean_obj_arg source, b_lean_obj_arg target) + { + const auto src = text(source); + const auto dst = text(target); + return io([&]() { + return traced(command("pending_messages", src, dst), [&]() { + return strings(driver->pending_messages(src, dst)); + }); + }); + } + lean_obj_res cft_dispatch_all_once() { return step( @@ -370,6 +421,16 @@ extern "C" }); } + lean_obj_res cft_drop_nth( + b_lean_obj_arg source, b_lean_obj_arg target, uint64_t position) + { + const auto src = text(source); + const auto dst = text(target); + return step(command("drop_nth", src, dst, position), [&]() { + driver->drop_nth(src, dst, position); + }); + } + lean_obj_res cft_shuffle_one(b_lean_obj_arg node) { const auto id = text(node); diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 93084099370..b3b8d7651d3 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -526,64 +526,50 @@ class CftDriver log(node_id, tgt_node_id, s, dropped); } - void log_msg_details( - ccf::NodeId node_id, - ccf::NodeId tgt_node_id, - const std::vector& contents, - bool dropped = false) + // Calls f with the packet at the start of a queued message + template + void visit_packet(const std::vector& contents, F&& f) { const uint8_t* data = contents.data(); size_t size = contents.size(); - const auto log_packet = [&](const auto& packet) { - log_msg_details(node_id, tgt_node_id, packet, dropped); - if (dropped) - { - aft::trace::drop_pending_to( - _nodes.at(node_id).cft->get_state_for_trace(), - node_id.value(), - tgt_node_id.value(), - packet); - } - }; - const auto msg_type = serialized::peek(data, size); switch (msg_type) { case (aft::CftMsgType::request_vote): { auto rv = *(aft::RequestVote*)data; - log_packet(rv); + f(rv); break; } case (aft::CftMsgType::request_pre_vote): { auto rpv = *(aft::RequestPreVote*)data; - log_packet(rpv); + f(rpv); break; } case (aft::CftMsgType::request_vote_response): { auto rvr = *(aft::RequestVoteResponse*)data; - log_packet(rvr); + f(rvr); break; } case (aft::CftMsgType::request_pre_vote_response): { auto rvr = *(aft::RequestPreVoteResponse*)data; - log_packet(rvr); + f(rvr); break; } case (aft::CftMsgType::append_entries): { auto ae = *(aft::AppendEntries*)data; - log_packet(ae); + f(ae); break; } case (aft::CftMsgType::append_entries_response): { auto aer = *(aft::AppendEntriesResponse*)data; - log_packet(aer); + f(aer); break; } case (aft::CftMsgType::append_entries_signed_response): @@ -594,7 +580,7 @@ class CftDriver case (aft::CftMsgType::propose_request_vote): { auto prv = *(aft::ProposeRequestVote*)data; - log_packet(prv); + f(prv); break; } default: @@ -605,6 +591,36 @@ class CftDriver } } + void log_msg_details( + ccf::NodeId node_id, + ccf::NodeId tgt_node_id, + const std::vector& contents, + bool dropped = false) + { + visit_packet(contents, [&](const auto& packet) { + log_msg_details(node_id, tgt_node_id, packet, dropped); + if (dropped) + { + aft::trace::drop_pending_to( + _nodes.at(node_id).cft->get_state_for_trace(), + node_id.value(), + tgt_node_id.value(), + packet); + } + }); + } + + // The packet of a queued message as JSON, with the fields which a trace + // records for it + std::string packet_json(const std::vector& contents) + { + std::vector encoded; + visit_packet(contents, [&encoded](const auto& packet) { + write_msgpack(encoded, packet); + }); + return nlohmann::json::from_msgpack(encoded).dump(); + } + void connect(ccf::NodeId first, ccf::NodeId second) { CFT_DRIVER_PRINT("{}-->{}: connect", first, second); @@ -926,6 +942,56 @@ class CftDriver } } + // The undelivered messages sent by src to dst, oldest first, each as + // packet_json + std::vector pending_messages(ccf::NodeId src, ccf::NodeId dst) + { + std::vector packets; + for (const auto& [target, contents] : + channel_stub_proxy(*_nodes.at(src).cft)->messages) + { + if (target == dst) + { + packets.push_back(packet_json(contents)); + } + } + return packets; + } + + // Removes and returns the undelivered message at position, counting from 0, + // among those sent by src to dst + std::vector take_pending( + ccf::NodeId src, ccf::NodeId dst, size_t position) + { + auto& messages = channel_stub_proxy(*_nodes.at(src).cft)->messages; + size_t seen = 0; + for (auto it = messages.begin(); it != messages.end(); ++it) + { + if (it->first == dst && seen++ == position) + { + auto contents = std::move(it->second); + messages.erase(it); + return contents; + } + } + + throw std::runtime_error(fmt::format( + "No undelivered message at position {} from {} to {}", + position, + src, + dst)); + } + + void dispatch_nth(ccf::NodeId src, ccf::NodeId dst, size_t position) + { + dispatch_single_message(src, dst, take_pending(src, dst, position)); + } + + void drop_nth(ccf::NodeId src, ccf::NodeId dst, size_t position) + { + log_msg_details(src, dst, take_pending(src, dst, position), true); + } + std::vector> find_primaries() { std::vector> primaries; diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 88bee4eeeb6..3b1b6061d99 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -127,6 +127,20 @@ opaque dispatchOne (node : @& NodeId) : IO Unit @[extern "cft_dispatch_single"] opaque dispatchSingle (source target : @& NodeId) : IO Unit +/-- +Deliver the message at `position`, counting from 0, among the undelivered messages sent by +`source` to `target`, oldest first. +-/ +@[extern "cft_dispatch_nth"] +opaque dispatchNth (source target : @& NodeId) (position : UInt64) : IO Unit + +/-- +The undelivered messages sent by `source` to `target`, oldest first. +Each is the JSON of its packet, with the fields which a trace records for it. +-/ +@[extern "cft_pending_messages"] +opaque pendingMessages (source target : @& NodeId) : IO (Array String) + /-- Deliver the messages already sent by every node, but not their responses. -/ @[extern "cft_dispatch_all_once"] opaque dispatchAllOnce : IO Unit @@ -143,6 +157,10 @@ opaque dropPending (node : @& NodeId) : IO Unit @[extern "cft_drop_pending_to"] opaque dropPendingTo (source target : @& NodeId) : IO Unit +/-- Drop the undelivered message at `position`, as `dispatchNth` counts it. -/ +@[extern "cft_drop_nth"] +opaque dropNth (source target : @& NodeId) (position : UInt64) : IO Unit + /-- Shuffle the undelivered messages sent by `node`. -/ @[extern "cft_shuffle_one"] opaque shuffleOne (node : @& NodeId) : IO Unit diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 5b179f27666..6f20f27ab07 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -41,6 +41,7 @@ import Scenarios.reconfig_0_1 import Scenarios.reconfiguration import Scenarios.reconnect import Scenarios.reelection +import Scenarios.reordered_messages import Scenarios.replicate import Scenarios.replicate_on_latest_leader import Scenarios.retire_backup diff --git a/tests/cft_scenarios/Scenarios/reordered_messages.lean b/tests/cft_scenarios/Scenarios/reordered_messages.lean new file mode 100644 index 00000000000..bd28e0dd4f0 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/reordered_messages.lean @@ -0,0 +1,94 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- Messages on one link overtake one another, or are lost one at a time. +-- A follower ignores entries below its commit index, and a primary ignores an +-- acknowledgement below its match index. +initialize scenario "reordered_messages" do + startNode 0 + replicate 2 [signature] + assertCommitIdx 0 2 + + trustNode 2 1 + replicate 2 [signature] + + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 4 + assertCommitIdx 1 4 + -- Initial network state [0, 1], all nodes in sync + + -- 0 sends 5 and 6, then a heartbeat + replicate 2 ["hello", signature] + periodicOne 0 10 + periodicOne 0 10 + assertPending 0 1 2 + + -- The heartbeat overtakes the entries, so 1 lacks its previous index + dispatchNth 0 1 1 + assertLastTxid 1 "2.4" + assertPending 0 1 1 + + -- 0 learns where the ledger of 1 ends, and sends 5 and 6 again + dispatchOne 1 + periodicOne 0 10 + assertPending 0 1 2 + + -- 1 receives one copy, and 0 commits + dispatchSingle 0 1 + dispatchOne 1 + assertCommitIdx 0 6 + + -- 1 learns of the commit from a heartbeat, which overtakes the other copy + periodicOne 0 10 + dispatchNth 0 1 1 + assertCommitIdx 1 6 + + -- The other copy is now below the commit index of 1, which ignores it + dispatchSingle 0 1 + assertPending 0 1 0 + assertPending 1 0 1 + dispatchOne 1 + + -- 0 sends 7 and 8, then 9 and 10, and 1 acknowledges each + replicate 2 ["world", signature] + periodicOne 0 10 + replicate 2 ["again", signature] + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "2.10" + assertPending 1 0 2 + + -- The later acknowledgement overtakes the earlier one, which 0 then ignores + dispatchNth 1 0 1 + assertCommitIdx 0 10 + dispatchSingle 1 0 + assertCommitIdx 0 10 + + -- 0 sends 11 and 12, then 13 and 14, and only the first message is lost + replicate 2 ["more", signature] + periodicOne 0 10 + replicate 2 ["last", signature] + periodicOne 0 10 + dropNth 0 1 0 + dispatchOne 0 + assertLastTxid 1 "2.10" + + -- 0 learns where the ledger of 1 ends, and sends from there + dispatchOne 1 + periodicOne 0 10 + dispatchOne 0 + dispatchOne 1 + assertCommitIdx 0 14 From 8614cdad49847be479aeb877fa9c36650417afec Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 22:19:22 +0000 Subject: [PATCH 03/31] Keep the entries of AppendEntries as sent in scenarios The driver read the entries of an AppendEntries from the ledger of the sender when the message was delivered. If the sender had rolled back by then, the receiver got its newer entries, or the message was abandoned. The host of a real node reads them when the message is sent. The driver now queues each AppendEntries with its entries as it is sent. No existing trace changes. The delayed_append_entries scenario delivers such a message late, and fails without this change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.h | 89 +++++++++++-------- tests/cft_scenarios/Scenarios.lean | 1 + .../Scenarios/delayed_append_entries.lean | 60 +++++++++++++ 3 files changed, 111 insertions(+), 39 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/delayed_append_entries.lean diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index b3b8d7651d3..32f492bf34a 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -8,8 +8,10 @@ #include "logging_stub.h" #include +#include #include #include +#include #include #include #include @@ -92,6 +94,32 @@ struct LoggingStubStore_Mermaid : public aft::LoggingStubStoreConfig } }; +// Queues each AppendEntries with its entries, read from the ledger of the +// sender as it is sent, like the host of a real node. A later rollback of the +// sender does not change them. The entries follow the packet in the message. +struct ChannelStubProxy_Entries : public aft::ChannelStubProxy +{ + std::function(const aft::AppendEntries&)> read_entries; + + bool send_consensus_message( + const ccf::NodeId& to, const uint8_t* data, size_t size) override + { + if (!aft::ChannelStubProxy::send_consensus_message(to, data, size)) + { + return false; + } + + if (serialized::peek(data, size) == aft::append_entries) + { + const auto entries = read_entries(*(aft::AppendEntries*)data); + auto& message = messages.back().second; + message.insert(message.end(), entries.begin(), entries.end()); + } + + return true; + } +}; + using ms = std::chrono::milliseconds; using TCft = aft::Aft; using Store = LoggingStubStore_Mermaid; @@ -212,13 +240,26 @@ class CftDriver { auto kv = std::make_shared(node_id); const ccf::consensus::Configuration settings{{"10ms"}, {"100ms"}}; + auto channels = std::make_shared(); auto cft = std::make_shared( settings, std::make_unique(kv), std::make_unique(node_id), - std::make_shared(), + channels, std::make_shared(node_id), nullptr); + channels->read_entries = + [ledger = cft->ledger.get()](const aft::AppendEntries& ae) { + auto entries = ledger->get_append_entries_payload(ae); + if (!entries.has_value()) + { + throw std::logic_error(fmt::format( + "Sent AppendEntries ({}, {}] is not in the ledger", + ae.prev_idx, + ae.idx)); + } + return std::move(*entries); + }; kv->set_set_retired_committed_hook( [cft](aft::Index idx, const std::vector& node_ids) { cft->set_retired_committed(idx, node_ids); @@ -803,51 +844,21 @@ class CftDriver { if (_connections.find(std::make_pair(src, dst)) != _connections.end()) { - // If this is AppendEntries, fetch the corresponding ledger entries. + // The entries of an AppendEntries follow its packet, since it was sent const uint8_t* data = contents.data(); auto size = contents.size(); auto msg_type = serialized::peek(data, size); - bool should_send = true; - std::vector body; + std::span packet = contents; + std::span entries; if (msg_type == aft::append_entries) { - // Parse the indices to be sent to the recipient. - auto ae = *(aft::AppendEntries*)data; - - auto& sender_cft = _nodes.at(src).cft; - auto payload_opt = sender_cft->ledger->get_append_entries_payload(ae); - - if (!payload_opt.has_value()) - { - // While trying to construct an AppendEntries, we asked for an - // entry that doesn't exist. This is a valid situation - we queued - // the AppendEntries, but rolled back before it was dispatched! - // We abandon this operation here. - // We could log this in Mermaid with the line below, but since - // this does not occur in a real node it is silently ignored. In a - // real node, the AppendEntries and truncate messages are ordered - // and processed by the host in that order. All AppendEntries - // referencing a specific index will be processed before any - // truncation that removes that index. - // CFT_DRIVER_PRINT( - // "Note right of {}: Abandoning AppendEntries" - // "containing {} - no longer in ledger", - // node_id, - // idx); - should_send = false; - } - else - { - body = std::move(*payload_opt); - } + entries = packet.subspan(sizeof(aft::AppendEntries)); + packet = packet.first(sizeof(aft::AppendEntries)); } - if (should_send) - { - log_msg_details(src, dst, contents); - _nodes.at(dst).cft->recv_message(src, contents, body); - return true; - } + log_msg_details(src, dst, contents); + _nodes.at(dst).cft->recv_message(src, packet, entries); + return true; } // Log that this message was dropped diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 6f20f27ab07..afbe9857f7c 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -10,6 +10,7 @@ import Scenarios.check_quorum_012_0 import Scenarios.check_quorum_0_012 import Scenarios.commit_before_pending_reconfiguration import Scenarios.commit_stepwise_through_reconfiguration +import Scenarios.delayed_append_entries import Scenarios.delayed_vote_requests import Scenarios.election import Scenarios.election_while_reconfiguration diff --git a/tests/cft_scenarios/Scenarios/delayed_append_entries.lean b/tests/cft_scenarios/Scenarios/delayed_append_entries.lean new file mode 100644 index 00000000000..344feddac71 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/delayed_append_entries.lean @@ -0,0 +1,60 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- An AppendEntries from a deposed primary is delivered after that primary has rolled +-- back the entry which it carries. The follower receives the entry as it was sent. +initialize scenario "delayed_append_entries" do + startNode 0 + replicate 2 [signature] + assertCommitIdx 0 2 + + trustNodes 2 [1, 2] + replicate 2 [signature] + + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 4 + assertStateSync + -- Initial network state [0, 1, 2], all nodes in sync + + -- 0 sends 5 to 1 and 2. 1 never receives it, and 2 not yet + replicate 2 ["hello"] + periodicOne 0 10 + dropPendingTo 0 1 + + -- 1 is elected with the vote of 0, and 2 hears nothing of it + periodicOne 1 100 + assertDetail 1 "leadership_state" "PreVoteCandidate" + dropPendingTo 1 2 + dispatchOne 1 + dispatchSingle 0 1 + assertDetail 1 "leadership_state" "Candidate" + dropPendingTo 1 2 + dispatchOne 1 + assertDetail 0 "leadership_state" "Follower" + dispatchSingle 0 1 + assertDetail 1 "leadership_state" "Leader" + dropPendingTo 1 2 + + -- 1 replicates another 5, and 6. 0 rolls back its own 5 to take them + replicate 3 ["world", signature] + dispatchSingle 1 0 + periodicOne 1 10 + dropPendingTo 1 2 + dispatchSingle 1 0 + assertLastTxid 0 "3.6" + + -- 2 is still in term 2, and now receives the 5 which 0 sent in that term + assertDetail 2 "current_view" 2 + dispatchSingle 0 2 + assertLastTxid 2 "2.5" + + -- 1 brings 2 up to date, which rolls back that 5 + loopUntilSync + assertLastTxid 2 "3.7" From 2ef00503da2f1eada1e440d3b869003c131918db Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 22:30:00 +0000 Subject: [PATCH 04/31] Return the trace of consensus scenarios to Lean startCapture starts the driver with its trace kept in memory, and takeTrace returns the records emitted since the last call, as JSON. A Lean program can then see what each command did, in the process which issued it. FluentdSink exports records only over TCP, so the driver points it at a local listener, which a thread reads. takeTrace waits for a marker record, because the sink sends from another thread. cft_driver takes a file after the scenario name, and writes the trace to it. For 58 of the 59 scenarios that file equals the trace which the scenario test collects. The trace of nominate_successor varies between runs, by either route. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lean/consensus/replay/README.md | 1 + src/consensus/aft/test/driver.cpp | 259 ++++++++++++++++++++++++++--- tests/cft_scenarios/CftDriver.lean | 50 ++++-- tests/cft_scenarios/README.md | 8 + 4 files changed, 288 insertions(+), 30 deletions(-) diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index 5036bd246ba..adc10345727 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -31,6 +31,7 @@ done The scenario runner writes the raw records to `consensus/raw/`. The Fluentd tag is `ccf.cft`; packet message names are unprefixed, for example `append_entries` and `request_vote`. +For one scenario, `/tmp/cft-build/cft_driver .ndjson` writes the same trace, without the scenario test or `msgpack`. ## Outcomes diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 92fc60e54c3..82e1eaeeec3 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -12,11 +12,22 @@ #include "tracing/fluentd_sink.h" +#include +#include +#include +#include #include #include +#include +#include #include +#include #include +#include +#include #include +#include +#include #include namespace @@ -139,6 +150,218 @@ namespace } return array; } + + // Reads a connected socket as a stream + class SocketBuffer : public std::streambuf + { + int fd; + std::array buffer = {}; + + protected: + int_type underflow() override + { + ssize_t count = 0; + do + { + count = ::recv(fd, buffer.data(), buffer.size(), 0); + } while (count < 0 && errno == EINTR); + + if (count <= 0) + { + return traits_type::eof(); + } + + setg(buffer.data(), buffer.data(), buffer.data() + count); + return traits_type::to_int_type(*gptr()); + } + + public: + explicit SocketBuffer(int fd_) : fd(fd_) {} + }; + + // Keeps the trace in this process, for cft_take_trace. FluentdSink exports + // records only over TCP, so it is pointed at a local listener, which a + // thread reads. + class TraceCapture + { + // take() emits a record with this key, to learn that the records before + // it have arrived + static constexpr const char* MARKER = "sync"; + + int listener = -1; + uint16_t listening_port = 0; + std::atomic peer = -1; + std::thread collector; + + std::mutex mutex; + std::condition_variable arrived; + // Records which have arrived and are not yet taken, as JSON + std::vector records; + uint64_t arrived_marker = 0; + uint64_t emitted_marker = 0; + bool closed = false; + std::string error; + + void collect() + { + try + { + peer = ::accept4(listener, nullptr, nullptr, SOCK_CLOEXEC); + if (peer < 0) + { + throw std::runtime_error( + fmt::format("Cannot accept its connection: errno {}", errno)); + } + + SocketBuffer buffer(peer); + std::istream stream(&buffer); + while (stream.peek() != std::istream::traits_type::eof()) + { + // A Fluentd forward message is [tag, time, record] + const auto message = nlohmann::json::from_msgpack(stream, false); + const auto& record = message.at(2); + const auto& msg = record.at("msg"); + + std::lock_guard lock(mutex); + if (msg.contains(MARKER)) + { + arrived_marker = msg.at(MARKER).get(); + arrived.notify_all(); + } + else + { + records.push_back(record.dump()); + } + } + } + catch (const std::exception& e) + { + std::lock_guard lock(mutex); + error = e.what(); + } + + std::lock_guard lock(mutex); + closed = true; + arrived.notify_all(); + } + + public: + TraceCapture() + { + sockaddr_in address = {}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + auto* generic = reinterpret_cast(&address); + socklen_t size = sizeof(address); + + listener = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + if ( + listener < 0 || ::bind(listener, generic, size) != 0 || + ::listen(listener, 1) != 0 || + ::getsockname(listener, generic, &size) != 0) + { + const auto reason = errno; + if (listener >= 0) + { + ::close(listener); + } + throw std::runtime_error( + fmt::format("Cannot listen for the trace: errno {}", reason)); + } + + listening_port = ntohs(address.sin_port); + collector = std::thread([this]() { collect(); }); + } + + ~TraceCapture() + { + // Unblock the collector, if the sink never connected or did not stop + ::shutdown(listener, SHUT_RDWR); + if (peer >= 0) + { + ::shutdown(peer, SHUT_RDWR); + } + if (collector.joinable()) + { + collector.join(); + } + if (peer >= 0) + { + ::close(peer); + } + ::close(listener); + } + + [[nodiscard]] uint16_t port() const + { + return listening_port; + } + + // The records emitted since the last call + std::vector take() + { + using Sink = ccf::tracing::FluentdSink; + if (Sink::is_running()) + { + // The sink sends from another thread, so wait for a marker to arrive + ++emitted_marker; + ccf::tracing::emit(aft::trace::cft_trace_tag, MARKER, emitted_marker); + + std::unique_lock lock(mutex); + arrived.wait_for(lock, std::chrono::seconds(10), [this]() { + return arrived_marker == emitted_marker || closed; + }); + if (arrived_marker != emitted_marker) + { + throw std::runtime_error(fmt::format( + "The trace did not arrive: {}", + closed ? "its connection closed" : "timed out")); + } + } + else if (collector.joinable()) + { + // The sink has sent what it held, and closed its connection + collector.join(); + } + + const auto dropped = Sink::dropped_count(); + std::lock_guard lock(mutex); + if (dropped != 0 || !error.empty()) + { + throw std::runtime_error(fmt::format( + "The trace is incomplete: {} events dropped. {}", dropped, error)); + } + return std::exchange(records, {}); + } + }; + + std::unique_ptr capture; + + // Exports the trace to the Fluentd collector at a host and port, if given, + // and creates the driver + void start(const std::vector& host_and_port) + { + // Log all CFT steps to stdout; cft_scenarios_runner.py + // filters them. + ccf::logger::config::add_text_console_logger(); + ccf::logger::config::level() = ccf::LoggerLevel::DEBUG; + + if (!host_and_port.empty()) + { + ccf::tracing::FluentdSink::Endpoint endpoint{ + host_and_port.at(0), host_and_port.at(1)}; + endpoint.tag_prefix_allow_list = {"ccf.cft"}; + ccf::tracing::FluentdSink::configure(endpoint); + ccf::tracing::FluentdSink::bind_producer(0); + if (!ccf::tracing::FluentdSink::wait_for_connection( + std::chrono::seconds(5))) + { + throw std::runtime_error("Timed out waiting for Fluentd connection"); + } + } + + driver = std::make_unique(); + } } extern "C" @@ -158,28 +381,26 @@ extern "C" lean_obj_res cft_start(b_lean_obj_arg fluentd) { - return io([&]() { - // Log all CFT steps to stdout; cft_scenarios_runner.py - // filters them. - ccf::logger::config::add_text_console_logger(); - ccf::logger::config::level() = ccf::LoggerLevel::DEBUG; + return io([&]() { start(texts(fluentd)); }); + } + + lean_obj_res cft_start_capture() + { + return io([]() { + capture = std::make_unique(); + start({"127.0.0.1", std::to_string(capture->port())}); + }); + } - const auto host_and_port = texts(fluentd); - if (!host_and_port.empty()) + lean_obj_res cft_take_trace() + { + return io([]() { + if (capture == nullptr) { - ccf::tracing::FluentdSink::Endpoint endpoint{ - host_and_port.at(0), host_and_port.at(1)}; - endpoint.tag_prefix_allow_list = {"ccf.cft"}; - ccf::tracing::FluentdSink::configure(endpoint); - ccf::tracing::FluentdSink::bind_producer(0); - if (!ccf::tracing::FluentdSink::wait_for_connection( - std::chrono::seconds(5))) - { - throw std::runtime_error("Timed out waiting for Fluentd connection"); - } + throw std::runtime_error( + "The trace is only kept in memory after start_capture"); } - - driver = std::make_unique(); + return strings(capture->take()); }); } diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 3b1b6061d99..7d90552561d 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -245,26 +245,54 @@ def scenario (name : String) (steps : IO Unit) : IO Unit := @[extern "cft_start"] private opaque start (fluentd : @& List String) : IO Unit +/-- +Start the driver, as `run` does before a scenario, and keep the trace in memory for +`takeTrace`. A process can start the driver once. +-/ +@[extern "cft_start_capture"] +opaque startCapture : IO Unit + +/-- +The trace records emitted since the last call, in order. +Each is JSON, like a line of a trace which the scenario test writes. +Only after `startCapture`. +-/ +@[extern "cft_take_trace"] +opaque takeTrace : IO (Array String) + +/-- +Confirm that the network can still make progress, as `run` does after a scenario, and +stop the driver. After `startCapture`, `takeTrace` then returns the rest of the trace. +-/ @[extern "cft_finish"] -private opaque finish : IO Unit +opaque finish : IO Unit /-- Run the scenario named by the first argument. -Any trace is exported to the Fluentd host and port given by the remaining arguments. +One more argument is a file, to which the trace is written. +Two more are a Fluentd host and port, to which the trace is exported. -/ def run (args : List String) : IO Unit := do - let usage := "Expected a scenario name, optionally followed by a Fluentd host and port" - let (name, fluentd) ← - match args with - | [name] => pure (name, []) - | [name, host, port] => pure (name, [host, port]) - | _ => throw <| IO.userError usage + let usage := + "Expected a scenario name, then optionally a trace file, or a Fluentd host and port" + let name :: target := args | throw <| IO.userError usage let registered ← scenarios.get let known := registered.map (·.1) |>.reverse let some steps := registered.lookup name | throw <| IO.userError s!"Unknown scenario '{name}'. Known scenarios: {known}" - start fluentd - steps - finish + match target with + | [file] => + startCapture + try + steps + finish + finally + let records ← takeTrace + IO.FS.writeFile file (String.join (records.toList.map (· ++ "\n"))) + | [] | [_, _] => + start target + steps + finish + | _ => throw <| IO.userError usage end CftDriver diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 24b68d49a94..7cedc518981 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -51,6 +51,12 @@ cmake --build build --target cft_driver build/cft_driver ``` +To also write the trace of the scenario to a file, name the file: + +```sh +build/cft_driver .ndjson +``` + `cft_scenario_test` runs every scenario in `Scenarios/` with [`cft_scenarios_runner.py`](../cft_scenarios_runner.py), which needs Python `msgpack`: ```sh @@ -65,3 +71,5 @@ The runner prints each scenario as a Mermaid sequence diagram, and writes its tr `lake` compiles the scenarios to a static library, which defines `main`. `cft_driver` links it with [`driver.cpp`](../../src/consensus/aft/test/driver.cpp), which implements each command declared in `CftDriver.lean` by calling `CftDriver` in [`driver.h`](../../src/consensus/aft/test/driver.h), and with the Lean runtime. A trace records each command as a `cmd` record, naming the command and its arguments. +The driver exports its trace over TCP, to a Fluentd collector such as the scenario test runs. +For a trace file, and for `takeTrace`, `driver.cpp` collects it from a local port in the same process. From be77db17faa6cc6f31b381698f2e432ea80294a7 Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 22:40:31 +0000 Subject: [PATCH 05/31] Replay consensus traces one driver command at a time A Session replays the records of each driver command as they arrive, and keeps the model state between commands, so that a program which issues commands can follow the model as it goes. consensus-replay --by-command replays a saved trace in this way, and exits with status 3 if replaying the whole trace gives another result. The Lean workflow replays the scenario traces and the fixtures both ways. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/README.md | 2 +- .github/workflows/lean-package.yml | 13 ++- .github/workflows/lean.yml | 1 + lean/consensus/Consensus.lean | 3 +- .../Consensus/Replay/Interpreter.lean | 27 +++-- .../consensus/Consensus/Replay/Reduction.lean | 35 +++++- lean/consensus/Consensus/Replay/Session.lean | 109 ++++++++++++++++++ .../Consensus/Tests/ReplaySession.lean | 75 ++++++++++++ lean/consensus/replay/README.md | 16 ++- lean/consensus/replay/ReplayMain.lean | 63 ++++++---- 10 files changed, 306 insertions(+), 38 deletions(-) create mode 100644 lean/consensus/Consensus/Replay/Session.lean create mode 100644 lean/consensus/Consensus/Tests/ReplaySession.lean diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 7ebc8d35a18..19af8fb2901 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -154,7 +154,7 @@ Caches are isolated by package and runner architecture and invalidated by change CCF package and common build outputs are not cached, and all verification steps still run. CCF consensus, KV, and consistency trace captures run in parallel with package checks. Three independent trace-validation jobs each build their own replayer, download the corresponding capture artifacts, and replay the traces without depending on the package matrix. -The CCF consensus replay job also adds to its summary the [outcomes](../../lean/consensus/replay/README.md#outcomes) of the model which the captured traces show, and those which none shows. +The CCF consensus replay job replays each trace whole and [by driver command](../../lean/consensus/replay/README.md#replay-by-command), and adds to its summary the [outcomes](../../lean/consensus/replay/README.md#outcomes) of the model which the captured traces show, and those which none shows. Consistency capture reuses the existing `consistency_trace_validation` e2e test, and its replay job also checks the consistency replay fixtures, including election and rollback traces. A final summary job requires every package and trace job to succeed. diff --git a/.github/workflows/lean-package.yml b/.github/workflows/lean-package.yml index 318225f2856..bca5bb80e59 100644 --- a/.github/workflows/lean-package.yml +++ b/.github/workflows/lean-package.yml @@ -112,6 +112,7 @@ jobs: for trace in lean/consensus/replay/fixtures/valid/*.ndjson; do echo "::group::Validate $trace" lean/consensus/.lake/build/bin/consensus-replay "$trace" + lean/consensus/.lake/build/bin/consensus-replay --by-command "$trace" echo "::endgroup::" done @@ -120,14 +121,18 @@ jobs: run: | set -euo pipefail shopt -s failglob - for trace in lean/consensus/replay/fixtures/invalid/*.ndjson; do - echo "::group::Reject $trace" + reject() { status=0 - lean/consensus/.lake/build/bin/consensus-replay "$trace" || status=$? + lean/consensus/.lake/build/bin/consensus-replay "$@" || status=$? if [ "$status" -ne 1 ]; then - echo "::error::$trace: expected rejection (exit 1), got exit $status" + echo "::error::$*: expected rejection (exit 1), got exit $status" exit 1 fi + } + for trace in lean/consensus/replay/fixtures/invalid/*.ndjson; do + echo "::group::Reject $trace" + reject "$trace" + reject --by-command "$trace" echo "::endgroup::" done diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index 55696fd901f..02fd6d3f1af 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -269,6 +269,7 @@ jobs: for trace in "$RUNNER_TEMP"/cft-scenario-traces/*.ndjson; do echo "::group::Validate $trace" lean/consensus/.lake/build/bin/consensus-replay "$trace" + lean/consensus/.lake/build/bin/consensus-replay --by-command "$trace" echo "::endgroup::" done diff --git a/lean/consensus/Consensus.lean b/lean/consensus/Consensus.lean index 45a7341c9df..6caccb98374 100644 --- a/lean/consensus/Consensus.lean +++ b/lean/consensus/Consensus.lean @@ -1,8 +1,9 @@ module public import Consensus.Properties -public import Consensus.Replay.Reduction +public import Consensus.Replay.Session import Consensus.Tests.CanonicalTraces import Consensus.Tests.ReplayDelivery import Consensus.Tests.ReplayOutcomes +import Consensus.Tests.ReplaySession diff --git a/lean/consensus/Consensus/Replay/Interpreter.lean b/lean/consensus/Consensus/Replay/Interpreter.lean index d6d489a681e..6567ec8bbbd 100644 --- a/lean/consensus/Consensus/Replay/Interpreter.lean +++ b/lean/consensus/Consensus/Replay/Interpreter.lean @@ -353,8 +353,8 @@ structure Result where outcomes : Array Outcome deriving Repr, BEq -def replay (header : Header) (instructions : Array Instruction) - : Except String Result := do +/-- The declared nodes are distinct and sorted, and the initial members are among them. -/ +def Header.validate (header : Header) : Except String Unit := do unless !header.declared.isEmpty && header.declared.all (!·.isEmpty) && header.declared.eraseDups == header.declared do @@ -366,15 +366,28 @@ def replay (header : Header) (instructions : Array Instruction) for node in header.bootstrap.sort do unless node < header.declared.length do throw s!"undeclared model node {node}" - if instructions.isEmpty then throw "replay instructions must not be empty" - let mut state : ReplayState := { state := header.initial } - let mut actions := 0 + +/-- Run instructions in order. Diagnostics number them from `first`. -/ +def runInstructions (header : Header) (state : ReplayState) + (instructions : Array Instruction) (first : Nat := 1) + : Except String ReplayState := do + let mut state := state for instruction in instructions, index in [:instructions.size] do state ← (runInstruction header state instruction).mapError fun error => - s!"instruction {index + 1} at {instruction.origin.label}: {error}" - if let .action .. := instruction then actions := actions + 1 + s!"instruction {first + index} at {instruction.origin.label}: {error}" + return state + +def countActions (instructions : Array Instruction) : Nat := + (instructions.filter (· matches .action ..)).size + +def replay (header : Header) (instructions : Array Instruction) + : Except String Result := do + header.validate + if instructions.isEmpty then throw "replay instructions must not be empty" + let state ← runInstructions header { state := header.initial } instructions + let actions := countActions instructions return { instructions := instructions.size, actions, diff --git a/lean/consensus/Consensus/Replay/Reduction.lean b/lean/consensus/Consensus/Replay/Reduction.lean index 4cbfcbaade8..66f93a68741 100644 --- a/lean/consensus/Consensus/Replay/Reduction.lean +++ b/lean/consensus/Consensus/Replay/Reduction.lean @@ -398,18 +398,45 @@ private def ruleStep : Reduction Unit := do <|> attempt ruleNomination <|> attempt ruleDrop -private def ruleTrace : Reduction String := do - let leader ← ruleBootstrap +/-- Every remaining event is in a group; an ungrouped event is never a stutter. -/ +private def ruleGroups : Reduction Unit := do many_ ruleStep if let some event ← CCF.Parsec.peek then failure s!"{event.record.location}: ungrouped event {event.function}" + +private def ruleTrace : Reduction String := do + let leader ← ruleBootstrap + ruleGroups return leader +private def runRule {α : Type} (rule : Reduction α) (events : Array Event) + (header : Header) (nextRequestId : ReqID) + : Except String (α × Array Instruction × ReqID) := do + let (result, state) ← CCF.Parsec.run events (rule.run { header, nextRequestId }) + return (result, state.instructions, state.nextRequestId) + /-- Consume every event exactly once; an ungrouped event is never a stutter. -/ def reduce (events : Array Event) (header : Header) : Except String (String × Array Instruction) := do - let (leader, state) ← CCF.Parsec.run events (ruleTrace.run { header }) - return (leader, state.instructions) + let (leader, instructions, _) ← runRule ruleTrace events header 1 + return (leader, instructions) + +/-- +Reduce the events of the first driver command of a trace, as `reduce` does those of a +whole trace. Also return the request ID from which the next command numbers transactions. +-/ +def reduceFirstCommand (events : Array Event) (header : Header) + : Except String (String × Array Instruction × ReqID) := + runRule ruleTrace events header 1 + +/-- +Reduce the events of a later driver command, which are whole native groups, numbering +transactions from `nextRequestId`. Also return the request ID for the next command. +-/ +def reduceCommand (events : Array Event) (header : Header) (nextRequestId : ReqID) + : Except String (Array Instruction × ReqID) := do + let ((), instructions, nextRequestId) ← runRule ruleGroups events header nextRequestId + return (instructions, nextRequestId) end Consensus.Replay.Reducer diff --git a/lean/consensus/Consensus/Replay/Session.lean b/lean/consensus/Consensus/Replay/Session.lean new file mode 100644 index 00000000000..54a6c886fc8 --- /dev/null +++ b/lean/consensus/Consensus/Replay/Session.lean @@ -0,0 +1,109 @@ +module + +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. + +public import Consensus.Replay.Reduction + +public section + +/-! +# Replay by command + +A driver runs one command at a time, and the events of each command are whole native groups. +A session replays the records of each command as they arrive, and keeps the model state +between commands, so that a program which issues commands can follow the model as it goes. +Replaying a trace by command gives the result of replaying it whole. +-/ + +namespace Consensus.Replay + +open Model.Node (ReqID) + +/-- A replay which has run the records of some commands, and awaits those of the next. -/ +structure Session where + header : Header + replay : ReplayState + /-- The request ID of the next transaction. -/ + nextRequestId : ReqID + /-- Instructions and actions run so far. -/ + instructions : Nat := 0 + actions : Nat := 0 + +namespace Session + +private def run (session : Session) (instructions : Array Instruction) + : Except String Session := do + let replay ← + runInstructions session.header session.replay instructions (session.instructions + 1) + return { + session with + replay + instructions := session.instructions + instructions.size + actions := session.actions + countActions instructions + } + +/-- +Start a replay among the `declared` nodes, with the records of the first command, which +bootstraps the network. +-/ +def start (declared : List String) (records : Array Record) : Except String Session := do + let (events, _) ← prepareEvents records + let header : Header := { bootstrap := ∅, declared } + let (leader, instructions, nextRequestId) ← Reducer.reduceFirstCommand events header + let header := { header with bootstrap := {← header.nodeId leader} } + header.validate + run { header, replay := { state := header.initial }, nextRequestId } instructions + +/-- Replay the records of the next command. -/ +def feed (session : Session) (records : Array Record) : Except String Session := do + let (events, _) ← prepareEvents records + let (instructions, nextRequestId) ← + Reducer.reduceCommand events session.header session.nextRequestId + run { session with nextRequestId } instructions + +/-- The model state after the commands replayed so far. -/ +def state (session : Session) : Model.State := + session.replay.state + +def result (session : Session) : Result where + instructions := session.instructions + actions := session.actions + observations := session.instructions - session.actions + outcomes := session.replay.shown + +end Session + +private def startsCommand (record : Record) : Bool := + ((optional record.value "msg").bind (optional · "cmd")).isSome + +/-- +The records of each driver command, in order, each starting with its `cmd` record. +Records before the first `cmd` record form a command of their own. +-/ +def splitCommands (records : Array Record) : Array (Array Record) := + records.foldl + (fun commands record => + if startsCommand record || commands.isEmpty then + commands.push #[record] + else + commands.modify (commands.size - 1) (·.push record)) + #[] + +/-- Replay a whole trace. -/ +def replayTrace (records : Array Record) : Except String Result := do + let (events, declared) ← prepareEvents records + let reduced ← reduce events declared + replay reduced.header reduced.instructions + +/-- Replay a trace one driver command at a time, among the nodes which it names. -/ +def replayByCommand (records : Array Record) : Except String Result := do + let (_, declared) ← prepareEvents records + let commands := splitCommands records + let some first := commands[0]? | throw "trace has no records" + let session ← Session.start declared first + let session ← (commands.extract 1).foldlM Session.feed session + if session.instructions == 0 then throw "replay instructions must not be empty" + return session.result + +end Consensus.Replay diff --git a/lean/consensus/Consensus/Tests/ReplaySession.lean b/lean/consensus/Consensus/Tests/ReplaySession.lean new file mode 100644 index 00000000000..90a99538d69 --- /dev/null +++ b/lean/consensus/Consensus/Tests/ReplaySession.lean @@ -0,0 +1,75 @@ +module + +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. + +meta import Consensus.Replay.Session + +public import Consensus.Replay.Session + +public section + +namespace Consensus.Tests.ReplaySession + +open Consensus Consensus.Replay +open CCF.MultiNodeTransitionSystem (nodeState) + +/-- The first node starts, then signs: two driver commands. -/ +private def trace : List String := + [ + r#"{"msg":{"cmd":"start_node,0"}}"#, + r#"{"msg":{"function":"become_leader","state":{"node_id":"0"}}}"#, + r#"{"msg":{"function":"replicate","state":{"node_id":"0"}}}"#, + r#"{"msg":{"function":"replicate_entry","state":{"node_id":"0"},"globally_committable":false}}"#, + r#"{"msg":{"function":"add_configuration","state":{"node_id":"0"},"nodes":{"0":{}}}}"#, + r#"{"msg":{"cmd":"replicate,2,$"}}"#, + r#"{"msg":{"function":"replicate","state":{"node_id":"0","last_idx":1,"commit_idx":0}}}"#, + r#"{"msg":{"function":"replicate_entry","state":{"node_id":"0"},"seqno":2,"globally_committable":true}}"#, + r#"{"msg":{"function":"commit","state":{"node_id":"0"},"idx":2}}"# + ] + +private def check (condition : Bool) (message : String) : IO Unit := do + unless condition do + throw (IO.userError message) + +/-- The ledger length and commit index of node 0 in the model. -/ +private def ledger (session : Session) : Option (Nat × Nat) := + (nodeState session.state 0).map fun node => (node.log.length, node.commitIndex) + +/-- Checks replay by command against replay of the whole trace, and the state between commands. -/ +def run : IO Unit := do + let records ← IO.ofExcept (parseLog "trace" ("\n".intercalate trace)) + let commands := splitCommands records + check (commands.map (·.size) == #[5, 4]) "split: each command has its own records" + match replayTrace records, replayByCommand records with + | .ok whole, .ok byCommand => + check (whole == byCommand && whole.actions == 1) + "by command: the result differs from that of the whole trace" + | whole, byCommand => + let describe (result : Except String Result) := + match result with + | .ok _ => "accepted" + | .error error => error + throw + (IO.userError s!"whole trace: {describe whole}; by command: {describe byCommand}") + let started ← IO.ofExcept (Session.start ["0"] commands[0]!) + check (ledger started == some (1, 0)) "start: the model has the first command only" + let signed ← IO.ofExcept (started.feed commands[1]!) + check (ledger signed == some (2, 2)) "feed: the model has the second command" + -- A command holds whole groups: an entry cannot follow its `replicate` in a later one. + match started.feed (commands[1]!.extract 0 2) >>= (·.feed (commands[1]!.extract 2)) with + | .error error => + check ((error.splitOn "ungrouped event replicate_entry").length == 2) + s!"split group: unexpected error {error}" + | .ok _ => throw (IO.userError "split group: a group was continued in the next command") + match Session.start ["1"] commands[0]! with + | .error error => + check ((error.splitOn "undeclared node '0'").length == 2) + s!"undeclared node: unexpected error {error}" + | .ok _ => + throw (IO.userError "undeclared node: a node outside the declared ones started") + IO.println "Replay session checks passed." + +#eval run + +end Consensus.Tests.ReplaySession diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index adc10345727..08ddbef10e6 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -53,6 +53,18 @@ To compare with the implementation, configure the scenario build with `-DCOVERAG (cd /tmp/cft-build && "$OLDPWD/../../scripts/coverage.sh" --show-uncovered cft_driver) ``` +## Replay by command + +The driver starts each command of a scenario with a `cmd` record, and the events which follow it are whole groups. +[`Session.lean`](../Consensus/Replay/Session.lean) replays the records of one command at a time, and keeps the model state between commands, so that a program which issues driver commands can follow the model as it goes. +To replay a saved trace in this way: + +```sh +./.lake/build/bin/consensus-replay --by-command path/to/trace.ndjson +``` + +This reports the trace as replay of the whole trace does, and exits with status 3 if that gives another result. + ## Replay contract Every protocol event must match a named rule; bootstrap selects the initial singleton membership. @@ -74,9 +86,11 @@ Native code does not trace the `propose_request_vote` send that follows `step_do - [`Reduction.lean`](../Consensus/Replay/Reduction.lean): named rules, explicit boundaries and rule-tagged snapshot checks. - [`Interpreter.lean`](../Consensus/Replay/Interpreter.lean): boundary-state field comparisons, packet selection, send checks and model execution. - [`Outcomes.lean`](../Consensus/Replay/Outcomes.lean): the outcomes of model steps, and their report. -- [`ReplayMain.lean`](ReplayMain.lean): CLI which validates one trace, or reports the outcomes of several. +- [`Session.lean`](../Consensus/Replay/Session.lean): replay of one driver command at a time. +- [`ReplayMain.lean`](ReplayMain.lean): CLI which validates one trace, whole or by command, or reports the outcomes of several. - [`ReplayDelivery.lean`](../Consensus/Tests/ReplayDelivery.lean): delivery selection among equal rendered packets. - [`ReplayOutcomes.lean`](../Consensus/Tests/ReplayOutcomes.lean): outcomes of chosen steps, and the report of several traces. +- [`ReplaySession.lean`](../Consensus/Tests/ReplaySession.lean): replay by command against replay of the whole trace, and the model state between commands. - [`cft_scenarios_runner.py`](../../../tests/cft_scenarios_runner.py): existing native scenario runner and capture generation. See [failure fixtures](fixtures/README.md) for examples and expected diagnostics. diff --git a/lean/consensus/replay/ReplayMain.lean b/lean/consensus/replay/ReplayMain.lean index e77be24eb9c..e2cb0f75b20 100644 --- a/lean/consensus/replay/ReplayMain.lean +++ b/lean/consensus/replay/ReplayMain.lean @@ -3,26 +3,57 @@ module -- Copyright (c) Microsoft Corporation. All rights reserved. -- Licensed under the Apache 2.0 License. -public import Consensus.Replay.Reduction +public import Consensus.Replay.Session public section open Lean Consensus.Replay -/-- Reduce one trace, and replay it against the model. -/ +/-- Replay one whole trace against the model. -/ private def validate (path : String) : IO (Except String Result) := do + return (← readLog path) >>= replayTrace + +private def accept (result : Result) : IO UInt32 := do + let status := + Json.mkObj + [ + ("status", toJson "ok"), + ("instructions", toJson result.instructions), + ("actions", toJson result.actions), + ("observations", toJson result.observations) + ] + IO.println status.compress + return 0 + +private def reject (path error : String) : IO UInt32 := do + IO.eprintln s!"{path}: {error}" + return 1 + +private def disagree (path reason : String) : IO UInt32 := do + IO.eprintln + s!"{path}: replay by command disagrees with replay of the whole trace: {reason}" + return 3 + +/-- +Replay one trace a driver command at a time. +Replay of the whole trace must give the same result. +-/ +private def validateByCommand (path : String) : IO UInt32 := do let records ← readLog path - return do - let (events, declared) ← prepareEvents (← records) - let reduced ← reduce events declared - replay reduced.header reduced.instructions + match records >>= replayTrace, records >>= replayByCommand with + | .ok whole, .ok byCommand => + if whole == byCommand then accept byCommand else disagree path "the results differ" + | .error _, .error error => reject path error + | .ok _, .error error => disagree path s!"only replay by command rejects it: {error}" + | .error error, .ok _ => + disagree path s!"only replay of the whole trace rejects it: {error}" /-- Validate each trace, then report the outcomes which they show together. -/ private def reportOutcomes (paths : List String) : IO UInt32 := do let mut traces := #[] for path in paths do match ← validate path with - | .error error => IO.eprintln s!"{path}: {error}"; return 1 + | .error error => return ← reject path error | .ok result => let name := (System.FilePath.mk path).fileStem.getD path traces := traces.push (name, result.outcomes) @@ -31,28 +62,20 @@ private def reportOutcomes (paths : List String) : IO UInt32 := do private def usage : IO UInt32 := do IO.eprintln "usage: consensus-replay TRACE.ndjson" + IO.eprintln " consensus-replay --by-command TRACE.ndjson" IO.eprintln " consensus-replay --outcomes TRACE.ndjson..." return 2 def main (args : List String) : IO UInt32 := do try match args with - | ["--outcomes"] => usage + | ["--outcomes"] | ["--by-command"] => usage | "--outcomes" :: paths => reportOutcomes paths + | ["--by-command", path] => validateByCommand path | [path] => match ← validate path with - | .error error => IO.eprintln s!"{path}: {error}"; return 1 - | .ok result => - let result := - Json.mkObj - [ - ("status", toJson "ok"), - ("instructions", toJson result.instructions), - ("actions", toJson result.actions), - ("observations", toJson result.observations) - ] - IO.println result.compress - return 0 + | .error error => reject path error + | .ok result => accept result | _ => usage catch error => IO.eprintln s!"{error}" From a4dc1c60839470958d8aefa9b696202f9e7f4ef8 Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 23:06:58 +0000 Subject: [PATCH 06/31] Run consensus scenarios with the Lean model in one process cft_model_driver links the scenario driver with the Lean model of the protocol. It runs a scenario and replays its trace, whole and by command, or runs commands chosen from the model: it replays each command as soon as the driver has run it, requires each undelivered message of the driver to be one which the model holds, delivers the messages of the model newest first, and asserts its commit indexes of the driver. It is a Lake package, tests/cft_model, which requires the scenarios and the consensus model. CMake builds it only with -DCFT_MODEL_DRIVER=ON, because lake then builds the model, and fetches the formatter which the package of the model requires. cft_model_test runs every scenario and the run in step, in the Lean workflow. The main of the scenarios moves to Main.lean, so that another program can import Scenarios. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/skills/testing/SKILL.md | 3 + .github/workflows/README.md | 3 +- .github/workflows/lean.yml | 10 +- .gitignore | 1 + CMakeLists.txt | 76 +++++++ doc/architecture/verification.rst | 5 +- .../Consensus/Replay/Interpreter.lean | 4 + lean/consensus/replay/README.md | 3 + scripts/lean-format-checks.sh | 8 +- tests/cft_model/ModelDriver.lean | 213 ++++++++++++++++++ tests/cft_model/README.md | 40 ++++ tests/cft_model/lake-manifest.json | 51 +++++ tests/cft_model/lakefile.toml | 18 ++ tests/cft_model/lean-toolchain | 1 + tests/cft_scenarios/Main.lean | 7 + tests/cft_scenarios/README.md | 5 +- tests/cft_scenarios/Scenarios.lean | 7 +- tests/cft_scenarios/lakefile.toml | 2 +- 18 files changed, 447 insertions(+), 10 deletions(-) create mode 100644 tests/cft_model/ModelDriver.lean create mode 100644 tests/cft_model/README.md create mode 100644 tests/cft_model/lake-manifest.json create mode 100644 tests/cft_model/lakefile.toml create mode 120000 tests/cft_model/lean-toolchain create mode 100644 tests/cft_scenarios/Main.lean diff --git a/.github/skills/testing/SKILL.md b/.github/skills/testing/SKILL.md index 5a8422d2810..5b7e766488c 100644 --- a/.github/skills/testing/SKILL.md +++ b/.github/skills/testing/SKILL.md @@ -27,6 +27,9 @@ Labels include `unit`, `e2e`, `partitions`, `perf`, `benchmark`, `cft_scenario`, `cft_scenario_test` runs the [consensus scenarios](../../../tests/cft_scenarios/README.md), Lean programs linked into `cft_driver`. If CMake cannot find `lake` when configuring, it does not build `cft_driver` and disables the test; install Lean and reconfigure rather than reporting the disabled test as passing. +`cft_model_test` runs the same scenarios with the [consensus model driver](../../../tests/cft_model/README.md), which replays each trace against the Lean model of the protocol in the process which records it. +CMake registers it only when configured with `-DCFT_MODEL_DRIVER=ON`. + The release workflow sets `CCF_USE_RELEASED_PYTHON_PACKAGE=1` so `tests.sh`, installed-package tests, and sandboxes install the latest released `ccf` package from PyPI with `uv pip install --upgrade --reinstall-package ccf ccf`, replacing any previously installed editable SDK. Without this flag, development and normal CI tests retain their editable local SDK installation. Installed sandboxes still default to the SDK version matching the installed CCF release. diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 19af8fb2901..3d59d1dc37e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -153,6 +153,7 @@ Each matrix job caches its third-party dependency checkouts and build outputs, l Caches are isolated by package and runner architecture and invalidated by changes to the Ubuntu version, Lean toolchain, package and common dependency manifests/configuration, or reusable workflow. CCF package and common build outputs are not cached, and all verification steps still run. CCF consensus, KV, and consistency trace captures run in parallel with package checks. +The CCF consensus capture job then builds the [consensus model driver](../../tests/cft_model/README.md) with `-DCFT_MODEL_DRIVER=ON`, and runs `cft_model_test`, which replays each scenario in the process which runs it. Three independent trace-validation jobs each build their own replayer, download the corresponding capture artifacts, and replay the traces without depending on the package matrix. The CCF consensus replay job replays each trace whole and [by driver command](../../lean/consensus/replay/README.md#replay-by-command), and adds to its summary the [outcomes](../../lean/consensus/replay/README.md#outcomes) of the model which the captured traces show, and those which none shows. Consistency capture reuses the existing `consistency_trace_validation` e2e test, and its replay job also checks the consistency replay fixtures, including election and rollback traces. @@ -185,7 +186,7 @@ The dedicated replay job builds `kv-replay` separately to validate the captured After the build, `scripts/lean-format-checks.sh` checks every tracked `.lean` file with the pinned leanfmt dependency. The workflow runs on pull requests that change `lean/` (except `lean/mathlib-equivalence/`), any `.lean` file, the formatter script, or the workflow. -It also runs on changes to the C++ sources and trace runners checked against the CCF consensus and KV models: `src/consensus/aft/`, the CCF consensus scenario tests, `src/kv/`, the public KV and transaction headers, and the KV trace runner. +It also runs on changes to the C++ sources and trace runners checked against the CCF consensus and KV models: `src/consensus/aft/`, the CCF consensus scenario tests and model driver, `src/kv/`, the public KV and transaction headers, and the KV trace runner. See the [local formatting commands](../../lean/disaster-recovery/README.md#formatting) to apply fixes. File: `lean.yml` diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index 02fd6d3f1af..a1c459e7b0b 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -9,6 +9,7 @@ on: - "tests/cft_trace*.py" - "tests/cft_scenarios*.py" - "tests/cft_scenarios/**" + - "tests/cft_model/**" - "src/consensus/aft/**" - "src/kv/**" - "include/ccf/kv/**" @@ -68,7 +69,7 @@ jobs: run: | set -euo pipefail git config --global --add safe.directory "$GITHUB_WORKSPACE" - cmake -S . -B "$RUNNER_TEMP/cft-build" -GNinja -DCMAKE_BUILD_TYPE=Debug -DVERBOSE_LOGGING=ON + cmake -S . -B "$RUNNER_TEMP/cft-build" -GNinja -DCMAKE_BUILD_TYPE=Debug -DVERBOSE_LOGGING=ON -DCFT_MODEL_DRIVER=ON cmake --build "$RUNNER_TEMP/cft-build" --target cft_driver --parallel "$(( ($(nproc) + 1) >> 1 ))" # RUNNER_TEMP can persist between jobs on reused self-hosted runners; drop files from earlier runs. rm -rf "$RUNNER_TEMP/cft-build/consensus/raw" @@ -84,6 +85,13 @@ jobs: retention-days: 1 overwrite: true + # cft_model_driver also links the consensus model, which lake builds. + - name: Run scenarios with the consensus model + run: | + set -euo pipefail + cmake --build "$RUNNER_TEMP/cft-build" --target cft_model_test --parallel "$(( ($(nproc) + 1) >> 1 ))" + ctest --test-dir "$RUNNER_TEMP/cft-build" -VV -R "^cft_model_test$" --no-tests=error + kv_traces: name: KV trace captures runs-on: [gha-vmss-d16av7-ci-c] diff --git a/.gitignore b/.gitignore index 152e0f2a1a6..88f8196d36c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ CMakeFiles/ libuv/ cft_tests.md tests/cft_scenarios/.lake/ +tests/cft_model/.lake/ *.pyc __pycache__/ sphinx/build/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 84d4b942137..deaae02c940 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1312,6 +1312,82 @@ if(BUILD_TESTS) ) endif() + # Consensus model driver: the scenario driver, linked also with the Lean + # model of the protocol, which replays the trace of a scenario in the + # process which runs it. Opt-in, because lake then builds the model, and + # fetches the formatter which the package of the model requires. + option( + CFT_MODEL_DRIVER + "Build cft_model_driver, which runs consensus scenarios with the Lean model" + OFF + ) + if(CFT_MODEL_DRIVER) + if(NOT LAKE) + message( + FATAL_ERROR + "CFT_MODEL_DRIVER needs lake. Install Lean (https://lean-lang.org/install/)." + ) + endif() + + set(CFT_MODEL_DIR ${CMAKE_SOURCE_DIR}/tests/cft_model) + set( + CFT_MODEL_LIBRARIES + ${CFT_MODEL_DIR}/.lake/build/lib/libcft__model_ModelDriver.a + ${CMAKE_SOURCE_DIR}/lean/consensus/.lake/build/lib/libconsensus_Consensus.a + ${CMAKE_SOURCE_DIR}/lean/common/.lake/build/lib/libccf__common_CCF.a + ) + add_custom_target( + cft_model + COMMAND + ${LAKE} build ModelDriver:static @consensus/Consensus:static + @ccf_common/CCF:static + WORKING_DIRECTORY ${CFT_MODEL_DIR} + BYPRODUCTS ${CFT_MODEL_LIBRARIES} + COMMENT "Building consensus model driver" + VERBATIM + ) + # lake builds the scenarios for this package too, so never at once + add_dependencies(cft_model cft_scenarios) + + add_executable( + cft_model_driver + ${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/aft/test/driver.cpp + ) + add_dependencies(cft_model_driver cft_model) + add_san(cft_model_driver) + add_warning_checks(cft_model_driver) + enable_coverage(cft_model_driver) + # As for cft_driver. The model driver library comes first, so that its + # main is linked, and not that of the scenario library. + target_link_libraries( + cft_model_driver + PRIVATE + ${CFT_MODEL_LIBRARIES} + ${CFT_SCENARIOS_LIBRARY} + ccfcrypto + ccf_tasks + gcc_s + ${LEAN_SYSROOT}/lib/lean/libleanshared.so + ) + target_include_directories(cft_model_driver PRIVATE src/consensus/aft) + target_include_directories( + cft_model_driver + SYSTEM + PRIVATE ${LEAN_SYSROOT}/include + ) + + add_test(NAME cft_model_test COMMAND cft_model_driver --all) + add_test_target(cft_model_test cft_model_driver) + if(COVERAGE) + # One process per scenario, as for cft_scenario_test + set_property( + TEST cft_model_test + APPEND + PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=cft_model-%p.profraw" + ) + endif() + endif() + add_test( NAME csr_test COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/certs.py ./kp_cert_test diff --git a/doc/architecture/verification.rst b/doc/architecture/verification.rst index cd6391ecbb3..ca69d508a66 100644 --- a/doc/architecture/verification.rst +++ b/doc/architecture/verification.rst @@ -63,6 +63,7 @@ Each replayer consumes traces produced by existing C++ tests: * **Consensus**: the ``cft_scenario_test`` test runs :ccf_repo:`tests/cft_scenarios_runner.py`, which executes each scenario in :ccf_repo:`tests/cft_scenarios/Scenarios/` with the ``cft_driver`` binary. The scenarios are Lean programs, which drive the implementation through the bindings in :ccf_repo:`tests/cft_scenarios/CftDriver.lean` and :ccf_repo:`src/consensus/aft/test/driver.cpp`; see :ccf_repo:`tests/cft_scenarios/README.md`. The driver exports consensus events as Fluentd records with the ``ccf.cft`` tag, which the runner collects and writes as NDJSON traces to ``consensus/raw/`` in the build directory. + The ``cft_model_test`` test, built with ``-DCFT_MODEL_DRIVER=ON``, runs the same scenarios with ``cft_model_driver``, which links the driver with the replayer and replays each trace in the process which records it; see :ccf_repo:`tests/cft_model/README.md`. * **KV**: building with ``-DCCF_KV_TRACING=ON`` enables test-only trace capture in ``kv_test``. :ccf_repo:`tests/kv_trace_validation.py` captures the purpose-built ``KV trace *`` test cases and a seeded concurrent fuzzer, then checks every trace with ``kv-replay`` unless ``--capture-only`` is passed. Tracing is disabled in normal builds. @@ -80,6 +81,7 @@ The :ccf_repo:`Lean workflow <.github/workflows/lean.yml>` runs on pull requests * A package job for each of ``common``, ``consensus``, ``consistency``, ``disaster-recovery`` and ``kv`` runs the kernel and axiom check (``lake check``), import coverage and architecture checks, formatting and linting, and, where applicable, the package's replay fixtures. * Separate capture jobs build CCF and record consensus, KV and consistency traces from the tests described above. + The consensus capture job also runs ``cft_model_test``. * Replay jobs build each replayer and validate every captured trace. The consensus replay job also reports which outcomes of the model's handlers the captured traces show, and which none shows; see :ccf_repo:`lean/consensus/replay/README.md`. * A final ``Lean models and proofs`` job succeeds only if all of the above succeed. @@ -93,7 +95,8 @@ Running locally Install Lean with `elan `__. The Lean toolchain is pinned by :ccf_repo:`lean/lean-toolchain`, and the models are independent of normal CCF builds. -The only part of a CCF build which needs Lean is ``cft_driver``; without Lean, it is not built and ``cft_scenario_test`` is disabled. +The only part of a default CCF build which needs Lean is ``cft_driver``; without Lean, it is not built and ``cft_scenario_test`` is disabled. +``cft_model_driver`` also needs Lean, and is built only with ``-DCFT_MODEL_DRIVER=ON``. From a package directory under ``lean/``, the common checks are: .. code-block:: bash diff --git a/lean/consensus/Consensus/Replay/Interpreter.lean b/lean/consensus/Consensus/Replay/Interpreter.lean index 6567ec8bbbd..c77e9981f8b 100644 --- a/lean/consensus/Consensus/Replay/Interpreter.lean +++ b/lean/consensus/Consensus/Replay/Interpreter.lean @@ -248,6 +248,10 @@ private def checkPacket (packet : Json) (message : Model.Node.Message) unless missing.isEmpty do throw s!"packet lacks fields {missing}" +/-- Whether a native packet is the rendering of a model message. -/ +def packetMatches (packet : Json) (message : Model.Node.Message) : Bool := + (checkPacket packet message).isOk + /-- The first matching envelope, else the first with the same `msg`, to diagnose a mismatch. -/ private def closest (packet : Json) (envelopes : List Envelope) : Option Envelope := envelopes.find? (fun envelope => (checkPacket packet envelope.payload).isOk) diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index 08ddbef10e6..e6ec1acc79d 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -65,6 +65,8 @@ To replay a saved trace in this way: This reports the trace as replay of the whole trace does, and exits with status 3 if that gives another result. +The [consensus model driver](../../../tests/cft_model/README.md) is such a program: it links the driver with replay, and replays each command of a run as soon as the driver has run it. + ## Replay contract Every protocol event must match a named rule; bootstrap selects the initial singleton membership. @@ -91,6 +93,7 @@ Native code does not trace the `propose_request_vote` send that follows `step_do - [`ReplayDelivery.lean`](../Consensus/Tests/ReplayDelivery.lean): delivery selection among equal rendered packets. - [`ReplayOutcomes.lean`](../Consensus/Tests/ReplayOutcomes.lean): outcomes of chosen steps, and the report of several traces. - [`ReplaySession.lean`](../Consensus/Tests/ReplaySession.lean): replay by command against replay of the whole trace, and the model state between commands. +- [`ModelDriver.lean`](../../../tests/cft_model/ModelDriver.lean): the consensus model driver, which replays the scenarios in the process which runs them. - [`cft_scenarios_runner.py`](../../../tests/cft_scenarios_runner.py): existing native scenario runner and capture generation. See [failure fixtures](fixtures/README.md) for examples and expected diagnostics. diff --git a/scripts/lean-format-checks.sh b/scripts/lean-format-checks.sh index 84d21a7804b..8da06ca49c2 100755 --- a/scripts/lean-format-checks.sh +++ b/scripts/lean-format-checks.sh @@ -36,8 +36,11 @@ for package in "${PACKAGES[@]}"; do exit 1 fi # The disaster recovery package also checks Lean files outside lean/. + # The consensus package checks the model driver, which imports its model. if [ "$package" == "disaster-recovery" ]; then - PATHSPEC=("*.lean" ":(exclude)lean/consensus/*" ":(exclude)lean/consistency/*" ":(exclude)lean/kv/*" ":(exclude)lean/mathlib-equivalence/*") + PATHSPEC=("*.lean" ":(exclude)lean/consensus/*" ":(exclude)lean/consistency/*" ":(exclude)lean/kv/*" ":(exclude)lean/mathlib-equivalence/*" ":(exclude)tests/cft_model/*") + elif [ "$package" == "consensus" ]; then + PATHSPEC=("lean/consensus/*.lean" "tests/cft_model/*.lean") else PATHSPEC=("lean/$package/*.lean") fi @@ -49,7 +52,8 @@ for package in "${PACKAGES[@]}"; do fi lake build "$library" # The consensus scenarios are a package outside lean/, without leanfmt. - if [ "$package" == "disaster-recovery" ]; then + # The model driver imports them. + if [ "$package" == "disaster-recovery" ] || [ "$package" == "consensus" ]; then lake --dir "$ROOT_DIR/tests/cft_scenarios" build export LEAN_PATH="$ROOT_DIR/tests/cft_scenarios/.lake/build/lib/lean" fi diff --git a/tests/cft_model/ModelDriver.lean b/tests/cft_model/ModelDriver.lean new file mode 100644 index 00000000000..28030617321 --- /dev/null +++ b/tests/cft_model/ModelDriver.lean @@ -0,0 +1,213 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import Consensus.Replay.Session +import Scenarios + +/-! +Runs consensus scenarios with the Lean model of the protocol in the same process. + +The driver keeps its trace in memory, and the model replays it: after a whole scenario, +or after each command of a run which reads the model to choose its next command. +-/ + +open Lean +open CftDriver +open Consensus Consensus.Replay +open CCF.MultiNodeTransitionSystem (nodeState) + +namespace ModelDriver + +private def fail {α : Type} (message : String) : IO α := + throw (IO.userError message) + +/-- Take the trace records emitted since the last call, numbered after the `taken` before. -/ +private def takeRecords (source : String) (taken : Nat) : IO (Array Record) := do + (← takeTrace).mapIdxM + fun index line => do + let value ← IO.ofExcept (Json.parse line) + return { file := source, line := taken + index + 1, value } + +/-- +Run a scenario with its trace kept in memory, then replay the trace against the model: +whole, and one command at a time, which must agree. +-/ +private def runScenario (name : String) (steps : IO Unit) : IO Result := do + startCapture + steps + finish + let records ← takeRecords name 0 + let whole ← IO.ofExcept (replayTrace records) + let byCommand ← IO.ofExcept (replayByCommand records) + unless whole == byCommand do + fail "replay by command disagrees with replay of the whole trace" + return whole + +/-- A run which replays each driver command in the model as soon as it is issued. -/ +private structure Run where + /-- The nodes which the run may start, in lexical order. -/ + nodes : List String + /-- The records taken so far. -/ + records : Array Record := #[] + /-- The replay, once a command has started the first node. -/ + session : Option Session := none + +/-- The nodes which have started, by name and model identifier. -/ +private def started (session : Session) : List (String × Nat) := + session.header.declared.zipIdx.filter + fun (_, id) => (nodeState session.state id).any fun node => !decide node.isFresh + +/-- +Each undelivered message of the driver must be a distinct envelope in the network of the +model. The model also keeps the messages which the driver has dropped. +-/ +private def checkPending (session : Session) : IO Unit := do + let nodes := started session + for (source, sourceId) in nodes do + for (target, targetId) in nodes do + if sourceId != targetId then + let mut queued := + session.state.network.filter + fun envelope => envelope.source == sourceId && envelope.target == targetId + for pending in ← pendingMessages source target do + let packet ← IO.ofExcept (Json.parse pending) + let some envelope := queued.find? (packetMatches packet ·.payload) + | fail s!"the model holds no message from {source} to {target} like {pending}" + queued := queued.erase envelope + +/-- +Deliver the message which the model holds as `envelope`, wherever it is among the +undelivered messages of the driver. +-/ +private def deliver (session : Session) (envelope : Model.Envelope) : IO Unit := do + let name (id : Nat) := session.header.declared[id]?.getD (toString id) + let source := name envelope.source + let target := name envelope.target + let wanted := (messageJson envelope.payload).compress + let pending ← (← pendingMessages source target).mapM (IO.ofExcept <| Json.parse ·) + let some position := pending.findIdx? (packetMatches · envelope.payload) + | fail s!"the driver holds no message from {source} to {target} like {wanted}" + dispatchNth source target position.toUInt64 + +namespace Run + +/-- Replay the records emitted since the last call. -/ +private def observe (run : Run) : IO Run := do + let records ← takeRecords "trace" run.records.size + let session ← + match run.session with + | none => IO.ofExcept (Session.start run.nodes records) + | some session => IO.ofExcept (session.feed records) + return { run with records := run.records ++ records, session := some session } + +/-- Issue a command, replay what it did, and compare the undelivered messages. -/ +private def step (run : Run) (command : IO Unit) : IO Run := do + command + let run ← run.observe + if let some session := run.session then + checkPending session + return run + +/-- Deliver the messages which the model holds, the newest first, until none is left. -/ +private def drain (run : Run) : IO Run := do + let mut run := run + for _ in [0:100] do + let some session := run.session | break + let some envelope := session.state.network.getLast? | break + run ← run.step (deliver session envelope) + return run + +end Run + +/-- +A run in step with the model. A primary sends to a new follower three times, and after +each the messages which the model holds are delivered, the newest first. The commit +indexes of the model are then asserted of the driver, and replay of the whole trace must +give the result of the replay in step. +-/ +private def runInStep : IO Result := do + startCapture + let mut run : Run := { nodes := ["0", "1"] } + for command + in [startNode 0, replicate 2 [signature], trustNode 2 1, replicate 2 [signature]] do + run ← run.step command + for _ in [0:3] do + run ← run.step (periodicOne 0 10) + run ← run.drain + let some session := run.session | fail "no command started a node" + unless session.state.network.isEmpty do + fail "the model still holds messages" + for (node, id) in started session do + let some state := nodeState session.state id | fail s!"the model has no node {node}" + run ← run.step (assertCommitIdx node state.commitIndex.toUInt64) + finish + run ← run.observe + let some session := run.session | fail "no command started a node" + let whole ← IO.ofExcept (replayTrace run.records) + unless session.result == whole do + fail "replay in step disagrees with replay of the whole trace" + return whole + +/-- The argument which selects the run in step, where a scenario is named. -/ +private def inStep : String := "--in-step" + +/-- +Run every scenario and the run in step, each in a process of its own, because a process +can start the driver once. +-/ +private def runAll : IO UInt32 := do + let app ← IO.appPath + let names := (← scenarios.get).map (·.1) |>.reverse + let mut failed := 0 + for name in names ++ [inStep] do + let child ← + IO.Process.spawn + { cmd := app.toString, args := #[name], stdout := .null, stderr := .piped } + let errors ← child.stderr.readToEnd + if (← child.wait) == 0 then + IO.println s!"{name}: ok" + else + failed := failed + 1 + IO.println s!"{name}: FAILED\n{errors}" + IO.println s!"{names.length + 1} runs, {failed} failed" + return if failed == 0 then 0 else 1 + +/-- Report the result of one run. -/ +private def report (name : String) (run : IO Result) : IO UInt32 := do + try + let result ← run + let status := + Json.mkObj + [ + ("status", toJson "ok"), + ("run", toJson name), + ("instructions", toJson result.instructions), + ("actions", toJson result.actions) + ] + IO.println status.compress + return 0 + catch error => + IO.eprintln s!"{name}: {error}" + return 1 + +end ModelDriver + +open ModelDriver in +/-- +Run the scenario named by the argument, `--in-step` for the run in step with the model, +or `--all` for every scenario and the run in step. +-/ +def main (args : List String) : IO UInt32 := do + match args with + | ["--all"] => runAll + | [name] => + if name == inStep then + report "in step" runInStep + else + match (← scenarios.get).lookup name with + | some steps => report name (runScenario name steps) + | none => + IO.eprintln s!"Unknown scenario '{name}'" + return 2 + | _ => + IO.eprintln "usage: cft_model_driver SCENARIO | --in-step | --all" + return 2 diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md new file mode 100644 index 00000000000..a20eec9e338 --- /dev/null +++ b/tests/cft_model/README.md @@ -0,0 +1,40 @@ +# Consensus model driver + +`cft_model_driver` runs the [consensus scenarios](../cft_scenarios/README.md) with the [Lean model](../../lean/consensus/README.md) of the protocol in the same process. +The driver keeps its trace in memory, and [replay](../../lean/consensus/replay/README.md) checks it against the model as the run goes, so a program can read the state of the model to choose its next command. + +## Runs + +```sh +build/cft_model_driver +build/cft_model_driver --in-step +build/cft_model_driver --all +``` + +- `` runs the scenario of that name, then replays its trace whole and [by command](../../lean/consensus/replay/README.md#replay-by-command), which must agree. +- `--in-step` runs commands chosen from the model. + It replays each command as soon as the driver has run it, and requires each undelivered message of the driver to be a message which the model holds. + It delivers the messages which the model holds, newest first, and asserts of the driver the commit indexes of the model. +- `--all` runs every scenario and the run in step, each in a process of its own, because a process can start the driver once. + +A run prints its status as JSON. +It exits with status 1 if a command fails or the model rejects the trace, and 2 if its arguments are wrong. + +## Building + +The model driver needs the same prerequisites as [`cft_driver`](../cft_scenarios/README.md#running-scenarios), and CMake builds it only on request: + +```sh +cmake -S . -B build -GNinja -DCFT_MODEL_DRIVER=ON +cmake --build build --target cft_model_driver +ctest --test-dir build -VV -R '^cft_model_test$' --no-tests=error +``` + +`cft_model_test` runs `cft_model_driver --all`. +For the model driver, `lake` builds the consensus model, and fetches from GitHub the formatter which the package of the model requires. + +## Bindings + +[`ModelDriver.lean`](ModelDriver.lean) imports the scenarios and [`Session.lean`](../../lean/consensus/Consensus/Replay/Session.lean), and defines `main`. +`lake` compiles it, the scenarios and the model to static libraries. +`cft_model_driver` links them with [`driver.cpp`](../../src/consensus/aft/test/driver.cpp) and the Lean runtime, as `cft_driver` does the scenarios. diff --git a/tests/cft_model/lake-manifest.json b/tests/cft_model/lake-manifest.json new file mode 100644 index 00000000000..7a22c6394ff --- /dev/null +++ b/tests/cft_model/lake-manifest.json @@ -0,0 +1,51 @@ +{ + "version": "1.3.0", + "packagesDir": ".lake/packages", + "packages": [ + { + "type": "path", + "scope": "", + "name": "consensus", + "manifestFile": "lake-manifest.json", + "inherited": false, + "dir": "../../lean/consensus", + "copy": false, + "configFile": "lakefile.toml" + }, + { + "type": "path", + "scope": "", + "name": "cft", + "manifestFile": "lake-manifest.json", + "inherited": false, + "dir": "../cft_scenarios", + "copy": false, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/duckki/leanfmt.git", + "type": "git", + "subDir": null, + "scope": "", + "rev": "65d69c8dc2dfd0aa69fd4132d48f38b7af9a57ef", + "name": "leanfmt", + "manifestFile": "lake-manifest.json", + "inputRev": "65d69c8dc2dfd0aa69fd4132d48f38b7af9a57ef", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "type": "path", + "scope": "", + "name": "ccf_common", + "manifestFile": "lake-manifest.json", + "inherited": true, + "dir": "../../lean/consensus/../common", + "copy": false, + "configFile": "lakefile.toml" + } + ], + "name": "cft_model", + "lakeDir": ".lake", + "fixedToolchain": false +} diff --git a/tests/cft_model/lakefile.toml b/tests/cft_model/lakefile.toml new file mode 100644 index 00000000000..f91bc8ab52c --- /dev/null +++ b/tests/cft_model/lakefile.toml @@ -0,0 +1,18 @@ +name = "cft_model" +version = "0.1.0" +moreLeanArgs = ["-DwarningAsError=true"] +leanOptions = { autoImplicit = false } +defaultTargets = ["ModelDriver"] + +[[require]] +name = "cft" +path = "../cft_scenarios" +copy = false + +[[require]] +name = "consensus" +path = "../../lean/consensus" +copy = false + +[[lean_lib]] +name = "ModelDriver" diff --git a/tests/cft_model/lean-toolchain b/tests/cft_model/lean-toolchain new file mode 120000 index 00000000000..c2c872908ec --- /dev/null +++ b/tests/cft_model/lean-toolchain @@ -0,0 +1 @@ +../../lean/lean-toolchain \ No newline at end of file diff --git a/tests/cft_scenarios/Main.lean b/tests/cft_scenarios/Main.lean new file mode 100644 index 00000000000..88239f670eb --- /dev/null +++ b/tests/cft_scenarios/Main.lean @@ -0,0 +1,7 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import Scenarios + +/-- Run one of the scenarios, named by the first argument. -/ +def main (args : List String) : IO Unit := + CftDriver.run args diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 7cedc518981..318e26ea76d 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -68,8 +68,11 @@ The runner prints each scenario as a Mermaid sequence diagram, and writes its tr ## Bindings -`lake` compiles the scenarios to a static library, which defines `main`. +`lake` compiles the scenarios to a static library, which defines `main` in [`Main.lean`](Main.lean). `cft_driver` links it with [`driver.cpp`](../../src/consensus/aft/test/driver.cpp), which implements each command declared in `CftDriver.lean` by calling `CftDriver` in [`driver.h`](../../src/consensus/aft/test/driver.h), and with the Lean runtime. A trace records each command as a `cmd` record, naming the command and its arguments. The driver exports its trace over TCP, to a Fluentd collector such as the scenario test runs. For a trace file, and for `takeTrace`, `driver.cpp` collects it from a local port in the same process. + +Another program can import `Scenarios` and define its own `main`. +The [consensus model driver](../cft_model/README.md) does, to run the scenarios with the Lean model of the protocol in the same process. diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index afbe9857f7c..387f0325e90 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -62,6 +62,7 @@ import Scenarios.suffix_collision_2 import Scenarios.suffix_collision_3 import Scenarios.swap_single_node -/-- Run one of the imported scenarios, named by the first argument. -/ -def main (args : List String) : IO Unit := - CftDriver.run args +/-! +Every scenario. Importing this module registers them, for `Main` and for other programs +which run scenarios. +-/ diff --git a/tests/cft_scenarios/lakefile.toml b/tests/cft_scenarios/lakefile.toml index 65e79ac29e6..cfbfb3e3b87 100644 --- a/tests/cft_scenarios/lakefile.toml +++ b/tests/cft_scenarios/lakefile.toml @@ -6,4 +6,4 @@ defaultTargets = ["Scenarios"] [[lean_lib]] name = "Scenarios" -roots = ["Scenarios", "CftDriver"] +roots = ["Main", "Scenarios", "CftDriver"] From 655272be25645e0742b0a598133e7604a3f2d7dc Mon Sep 17 00:00:00 2001 From: achamayou Date: Fri, 13 Nov 2026 23:22:31 +0000 Subject: [PATCH 07/31] Replay nominations without a successor and ignored responses Two steps of the consensus scenarios ran no model action in replay, so no trace showed their outcomes. A primary which finds no successor traces a nomination and nothing else. Replay now runs the nomination of the model when the snapshot of a nomination is of a primary, so the model must find no successor either. CCF returned from an AppendEntries response of an unknown node before tracing it, because the event records the indexes which it keeps for the node. It now traces recv_append_entries_response_unknown_node, and replay delivers the response to the model, which decides by itself whether it knows the sender. The scenarios now show 69 of the 79 outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../consensus/Consensus/Replay/Reduction.lean | 16 +++++++++- .../Consensus/Tests/ReplaySession.lean | 31 ++++++++++++++++++- lean/consensus/replay/README.md | 7 +++-- src/consensus/aft/cft.h | 2 ++ src/consensus/aft/cft_trace_msgpack.h | 7 +++++ 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/lean/consensus/Consensus/Replay/Reduction.lean b/lean/consensus/Consensus/Replay/Reduction.lean index 66f93a68741..34b06fe557b 100644 --- a/lean/consensus/Consensus/Replay/Reduction.lean +++ b/lean/consensus/Consensus/Replay/Reduction.lean @@ -242,6 +242,17 @@ private def ruleRecvAppendEntriesResponse : Reduction Unit := do emitAction head "recv-append-entries-response" action emitObserveSends head (← extractSends start) +/-- CCF ignores the response of a node which it does not know. -/ +private def ruleRecvAppendEntriesResponseUnknownNode : Reduction Unit := do + let head ← peek + let action ← readReceive head ["append_entries_response"] + -- consume + event head "recv_append_entries_response_unknown_node" + -- emit + emitObserve head "recv-append-entries-response-unknown-node" + emitAction head "recv-append-entries-response-unknown-node" action + emitObserveSends head [] + private def ruleRecvRequestVote : Reduction Unit := do let head ← peek let action ← readReceive head ["request_vote", "request_pre_vote"] @@ -348,9 +359,11 @@ private def ruleNomination : Reduction Unit := do let start ← position event head "nomination" let nominated ← CCF.Parsec.optionalRule (event head "step_down_and_nominate_successor") + -- A primary which finds no successor also nominates, and sends no proposal. + let primary := optional head.state "leadership_state" == some (toJson "Leader") -- emit emitObserve head "nomination" - if nominated.isSome then + if nominated.isSome || primary then emitAction head "nomination" (.local head.node .nominateSuccessor) emitObserveSends head (← extractSends start) @@ -388,6 +401,7 @@ private def ruleStep : Reduction Unit := do -- Recv rules attempt ruleRecvAppendEntries <|> attempt ruleRecvAppendEntriesResponse + <|> attempt ruleRecvAppendEntriesResponseUnknownNode <|> attempt ruleRecvRequestVote <|> attempt ruleRecvProposeRequestVote <|> attempt ruleRecvRequestVoteResponse diff --git a/lean/consensus/Consensus/Tests/ReplaySession.lean b/lean/consensus/Consensus/Tests/ReplaySession.lean index 90a99538d69..6970f761c78 100644 --- a/lean/consensus/Consensus/Tests/ReplaySession.lean +++ b/lean/consensus/Consensus/Tests/ReplaySession.lean @@ -28,6 +28,14 @@ private def trace : List String := r#"{"msg":{"function":"commit","state":{"node_id":"0"},"idx":2}}"# ] +/-- Node 0, which has no peer, is asked to nominate a successor. -/ +private def nomination : String := + r#"{"msg":{"function":"nomination","state":{"node_id":"0","leadership_state":"Leader"}}}"# + +/-- Node 0 receives an acknowledgement from node 1, which it does not know. -/ +private def unknownNode : String := + r#"{"msg":{"function":"recv_append_entries_response_unknown_node","state":{"node_id":"0"},"from_node_id":"1","packet":{"msg":"append_entries_response","term":2,"last_log_idx":2,"success":"OK"}}}"# + private def check (condition : Bool) (message : String) : IO Unit := do unless condition do throw (IO.userError message) @@ -36,7 +44,10 @@ private def check (condition : Bool) (message : String) : IO Unit := do private def ledger (session : Session) : Option (Nat × Nat) := (nodeState session.state 0).map fun node => (node.log.length, node.commitIndex) -/-- Checks replay by command against replay of the whole trace, and the state between commands. -/ +/-- +Checks replay by command against replay of the whole trace, the state between commands, +and the commands which the model decides how to handle. +-/ def run : IO Unit := do let records ← IO.ofExcept (parseLog "trace" ("\n".intercalate trace)) let commands := splitCommands records @@ -68,6 +79,24 @@ def run : IO Unit := do s!"undeclared node: unexpected error {error}" | .ok _ => throw (IO.userError "undeclared node: a node outside the declared ones started") + -- A primary which finds no successor nominates in the model too. + let nominated ← IO.ofExcept (parseLog "nomination" nomination >>= signed.feed) + check (nominated.result.outcomes.contains .nominationWithoutSuccessor) + "nomination: the model was not asked for a successor" + -- The model also receives a response which CCF ignores, from a node it does not know. + let response : Model.Envelope := + { + source := 1 + target := 0 + payload := .appendEntriesResponse { term := 2, success := true, lastLogIndex := 2 } + } + let holding ← IO.ofExcept (Session.start ["0", "1"] commands[0]!) + let holding := { holding with replay.state.network := [response] } + let received ← IO.ofExcept (parseLog "unknown node" unknownNode >>= holding.feed) + check + (received.state.network.isEmpty + && received.result.outcomes.contains .appendEntriesResponseFromUnknownPeer) + "unknown node: the model did not receive the response" IO.println "Replay session checks passed." #eval run diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index e6ec1acc79d..02b9101c00e 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -44,7 +44,7 @@ To report how many steps and traces show each outcome, and which outcomes no tra ``` An outcome which no trace shows is a missing scenario, a step which the driver or the reduction cannot produce, or behaviour which the model allows and the implementation lacks. -For example, a nomination without a successor traces no `step_down_and_nominate_successor`, so replay runs no model action for it. +For example, the driver starts every node with an empty ledger, so no trace of a scenario shows `joinFromSnapshot`. To compare with the implementation, configure the scenario build with `-DCOVERAGE=ON` and run the scenario test. [`coverage.sh`](../../../scripts/coverage.sh) then lists the lines of [`cft.h`](../../../src/consensus/aft/cft.h) which the scenarios do not execute: @@ -81,6 +81,9 @@ It delivers the queued **model payload**, not reconstructing unrecorded native b An envelope that is never delivered stays queued, which models loss, so `drop_pending_to` emits no observation or action. After each rule's actions, the envelopes they queued must equal the rule's native sends, as a multiset of target and rendered packet. Native code does not trace the `propose_request_vote` send that follows `step_down_and_nominate_successor`, so replay counts each `step_down_and_nominate_successor` as that send. +A `nomination` whose snapshot is of a primary runs the model's nomination even if no `step_down_and_nominate_successor` follows it, so the model must find no successor either. +CCF ignores an AppendEntries response from a node which it does not know, and traces it as `recv_append_entries_response_unknown_node`. +Replay delivers that response too, and the model decides by itself whether it knows the sender. ## Source and checks @@ -92,7 +95,7 @@ Native code does not trace the `propose_request_vote` send that follows `step_do - [`ReplayMain.lean`](ReplayMain.lean): CLI which validates one trace, whole or by command, or reports the outcomes of several. - [`ReplayDelivery.lean`](../Consensus/Tests/ReplayDelivery.lean): delivery selection among equal rendered packets. - [`ReplayOutcomes.lean`](../Consensus/Tests/ReplayOutcomes.lean): outcomes of chosen steps, and the report of several traces. -- [`ReplaySession.lean`](../Consensus/Tests/ReplaySession.lean): replay by command against replay of the whole trace, and the model state between commands. +- [`ReplaySession.lean`](../Consensus/Tests/ReplaySession.lean): replay by command against replay of the whole trace, the model state between commands, and a nomination and a response which the model decides how to handle. - [`ModelDriver.lean`](../../../tests/cft_model/ModelDriver.lean): the consensus model driver, which replays the scenarios in the process which runs them. - [`cft_scenarios_runner.py`](../../../tests/cft_scenarios_runner.py): existing native scenario runner and capture generation. diff --git a/src/consensus/aft/cft.h b/src/consensus/aft/cft.h index c6e79fe9291..555b4059102 100644 --- a/src/consensus/aft/cft.h +++ b/src/consensus/aft/cft.h @@ -1542,6 +1542,8 @@ namespace aft if (node == all_other_nodes.end()) { // Ignore if we don't recognise the node. + trace::recv_append_entries_response_unknown_node( + *state, r, from.value()); CFT_FAIL_FMT( "Recv append entries response to {} from {}: unknown node", state->node_id, diff --git a/src/consensus/aft/cft_trace_msgpack.h b/src/consensus/aft/cft_trace_msgpack.h index 04c9863a513..01be61a3d4e 100644 --- a/src/consensus/aft/cft_trace_msgpack.h +++ b/src/consensus/aft/cft_trace_msgpack.h @@ -158,6 +158,13 @@ namespace aft::trace from_node_id, match_idx, sent_idx); + // Received from a node with no match_idx or sent_idx, and ignored + DECLARE_TRACE_EVENT( + recv_append_entries_response_unknown_node, + cft_trace_tag, + state, + packet, + from_node_id); DECLARE_TRACE_EVENT( recv_append_entries, cft_trace_tag, state, packet, from_node_id); DECLARE_TRACE_EVENT( From f3baec48f678ddcfe5ea9ad6952a557f4d14bd81 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 10:12:17 +0000 Subject: [PATCH 08/31] Let consensus scenarios hold a message back A scenario could deliver the messages of a node in order, or pick one to deliver or drop, but not delay one past a command such as dispatch_all, which delivers every message. hold_nth sets an undelivered message aside, where no other command delivers or drops it, and dispatch_held delivers it, however late. The model keeps the message in its network until then, so replay needs no change. held_messages and undelivered_messages list such messages with their sender and target, for a program which chooses among them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.cpp | 28 +++++++++ src/consensus/aft/test/driver.h | 92 ++++++++++++++++++++++++++++-- tests/cft_scenarios/CftDriver.lean | 26 +++++++++ tests/cft_scenarios/README.md | 1 + 4 files changed, 143 insertions(+), 4 deletions(-) diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 82e1eaeeec3..feed49ad83d 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -652,6 +652,34 @@ extern "C" }); } + lean_obj_res cft_undelivered_messages() + { + return io([]() { return strings(driver->undelivered_messages()); }); + } + + lean_obj_res cft_hold_nth( + b_lean_obj_arg source, b_lean_obj_arg target, uint64_t position) + { + const auto src = text(source); + const auto dst = text(target); + return step( + command("hold_nth", src, dst, position), + [&]() { driver->hold_nth(src, dst, position); }, + Invariants::SKIP); + } + + lean_obj_res cft_held_messages() + { + return io([]() { return strings(driver->held_messages()); }); + } + + lean_obj_res cft_dispatch_held(uint64_t position) + { + return step(command("dispatch_held", position), [&]() { + driver->dispatch_held(position); + }); + } + lean_obj_res cft_shuffle_one(b_lean_obj_arg node) { const auto id = text(node); diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 32f492bf34a..54516f37dbf 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -139,8 +139,18 @@ class CftDriver std::shared_ptr cft; }; + // A message which hold_nth has set aside + struct HeldMessage + { + ccf::NodeId src; + ccf::NodeId dst; + std::vector contents; + }; + std::map _nodes; std::set> _connections; + // Oldest first. Only dispatch_held delivers these. + std::vector _held; void _replicate( const std::string& term_s, @@ -651,15 +661,34 @@ class CftDriver }); } - // The packet of a queued message as JSON, with the fields which a trace - // records for it - std::string packet_json(const std::vector& contents) + // The packet of a queued message, with the fields which a trace records for + // it + nlohmann::json packet_fields(const std::vector& contents) { std::vector encoded; visit_packet(contents, [&encoded](const auto& packet) { write_msgpack(encoded, packet); }); - return nlohmann::json::from_msgpack(encoded).dump(); + return nlohmann::json::from_msgpack(encoded); + } + + // The packet of a queued message as JSON + std::string packet_json(const std::vector& contents) + { + return packet_fields(contents).dump(); + } + + // A queued message as JSON: its sender, its target and its packet + std::string message_json( + const ccf::NodeId& src, + const ccf::NodeId& dst, + const std::vector& contents) + { + nlohmann::json message; + message["from"] = src.value(); + message["to"] = dst.value(); + message["packet"] = packet_fields(contents); + return message.dump(); } void connect(ccf::NodeId first, ccf::NodeId second) @@ -1003,6 +1032,61 @@ class CftDriver log_msg_details(src, dst, take_pending(src, dst, position), true); } + // The undelivered messages of every node, each oldest first, as + // message_json + std::vector undelivered_messages() + { + std::vector messages; + for (const auto& [src, node] : _nodes) + { + for (const auto& [dst, contents] : + channel_stub_proxy(*node.cft)->messages) + { + messages.push_back(message_json(src, dst, contents)); + } + } + return messages; + } + + // Sets aside the undelivered message which dispatch_nth would deliver. It + // is no longer undelivered: no other step delivers or drops it. + void hold_nth(ccf::NodeId src, ccf::NodeId dst, size_t position) + { + auto contents = take_pending(src, dst, position); + CFT_DRIVER_PRINT( + "Note right of {}: Holding {} to {}", + src, + get_message_summary(contents), + dst); + _held.push_back({src, dst, std::move(contents)}); + } + + // The messages which hold_nth has set aside, oldest first, as message_json + std::vector held_messages() + { + std::vector messages; + for (const auto& held : _held) + { + messages.push_back(message_json(held.src, held.dst, held.contents)); + } + return messages; + } + + // Delivers the held message at position, counting from 0, or drops it if + // its sender and target are no longer connected + void dispatch_held(size_t position) + { + if (position >= _held.size()) + { + throw std::runtime_error( + fmt::format("No held message at position {}", position)); + } + + auto held = std::move(_held.at(position)); + _held.erase(_held.begin() + static_cast(position)); + dispatch_single_message(held.src, held.dst, std::move(held.contents)); + } + std::vector> find_primaries() { std::vector> primaries; diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 7d90552561d..d40b2229f8f 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -161,6 +161,32 @@ opaque dropPendingTo (source target : @& NodeId) : IO Unit @[extern "cft_drop_nth"] opaque dropNth (source target : @& NodeId) (position : UInt64) : IO Unit +/-- +The undelivered messages of every node, each oldest first. +Each is JSON, with the sender as `from`, the target as `to`, and the `packet` as +`pendingMessages` gives it. +-/ +@[extern "cft_undelivered_messages"] +opaque undeliveredMessages : IO (Array String) + +/-- +Set aside the undelivered message at `position`, as `dispatchNth` counts it. +It is then no longer undelivered: only `dispatchHeld` delivers it, however late. +-/ +@[extern "cft_hold_nth"] +opaque holdNth (source target : @& NodeId) (position : UInt64) : IO Unit + +/-- The messages which `holdNth` has set aside, oldest first, as `undeliveredMessages`. -/ +@[extern "cft_held_messages"] +opaque heldMessages : IO (Array String) + +/-- +Deliver the message at `position`, counting from 0, among those which `holdNth` has set +aside, oldest first. It is dropped if its sender and target are disconnected. +-/ +@[extern "cft_dispatch_held"] +opaque dispatchHeld (position : UInt64) : IO Unit + /-- Shuffle the undelivered messages sent by `node`. -/ @[extern "cft_shuffle_one"] opaque shuffleOne (node : @& NodeId) : IO Unit diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 318e26ea76d..340a91e2ea7 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -28,6 +28,7 @@ initialize scenario "example" do [`CftDriver.lean`](CftDriver.lean) documents every command. Node identifiers and terms are written as numbers or strings, and `latest` is the primary in the highest term. Each `replicate` appends one batch of entries, which are transaction data or `signature`. +A scenario delivers the messages of a node in the order in which they were sent, or picks one: `dispatchNth` delivers it, `dropNth` loses it, and `holdNth` sets it aside until `dispatchHeld` delivers it, however late. Scenarios are ordinary Lean, so they can also use loops, conditions and definitions. After each command which can alter state, the driver asserts that the commit index of every node is safe. From 52a09951764a5ef4a07a82d068e65b89b330f5aa Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 10:14:44 +0000 Subject: [PATCH 09/31] Explore changes to the consensus scenarios cft_model_driver --explore takes the commands of each scenario from its trace, changes them at random, and runs each changed program in a process of its own, which checks the run against the model. A change holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, or leaves out, repeats or swaps commands. A run is kept, to be changed in turn, if the model shows an outcome, or the run reaches an edge of cft.h, which no run before it has, and does so again when it is run a second time. The driver counts the edges of aft::Aft with -fsanitize-coverage. A run in which the driver fails, or the model rejects the trace, is a finding, reported once for each message, numbers aside. --record writes the commands of a scenario as a program, and --run runs a program. --shrink leaves out the steps which a finding or an outcome does not need, and --scenario writes a program as a scenario in Lean. cft_model_test also runs each scenario from its recorded commands, which must show the same outcomes, and explores briefly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CMakeLists.txt | 16 + doc/architecture/verification.rst | 1 + lean/consensus/replay/README.md | 1 + scripts/lean-format-checks.sh | 5 + src/consensus/aft/test/driver.cpp | 28 + tests/cft_model/ModelDriver.lean | 143 +++-- tests/cft_model/ModelDriver/Command.lean | 323 ++++++++++++ tests/cft_model/ModelDriver/Episode.lean | 451 ++++++++++++++++ tests/cft_model/ModelDriver/Explore.lean | 631 +++++++++++++++++++++++ tests/cft_model/README.md | 54 ++ tests/cft_model/edges.allowlist | 8 + 11 files changed, 1612 insertions(+), 49 deletions(-) create mode 100644 tests/cft_model/ModelDriver/Command.lean create mode 100644 tests/cft_model/ModelDriver/Episode.lean create mode 100644 tests/cft_model/ModelDriver/Explore.lean create mode 100644 tests/cft_model/edges.allowlist diff --git a/CMakeLists.txt b/CMakeLists.txt index deaae02c940..1988ea36e0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1357,6 +1357,22 @@ if(BUILD_TESTS) add_san(cft_model_driver) add_warning_checks(cft_model_driver) enable_coverage(cft_model_driver) + # Count the edges of the consensus implementation which a run reaches, + # so that the model driver can tell the runs which reach new ones + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CFT_MODEL_EDGES ${CFT_MODEL_DIR}/edges.allowlist) + target_compile_options( + cft_model_driver + PRIVATE + -fsanitize-coverage=inline-8bit-counters + -fsanitize-coverage-allowlist=${CFT_MODEL_EDGES} + ) + set_property( + SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/aft/test/driver.cpp + APPEND + PROPERTY OBJECT_DEPENDS ${CFT_MODEL_EDGES} + ) + endif() # As for cft_driver. The model driver library comes first, so that its # main is linked, and not that of the scenario library. target_link_libraries( diff --git a/doc/architecture/verification.rst b/doc/architecture/verification.rst index ca69d508a66..2453029aa33 100644 --- a/doc/architecture/verification.rst +++ b/doc/architecture/verification.rst @@ -64,6 +64,7 @@ Each replayer consumes traces produced by existing C++ tests: The scenarios are Lean programs, which drive the implementation through the bindings in :ccf_repo:`tests/cft_scenarios/CftDriver.lean` and :ccf_repo:`src/consensus/aft/test/driver.cpp`; see :ccf_repo:`tests/cft_scenarios/README.md`. The driver exports consensus events as Fluentd records with the ``ccf.cft`` tag, which the runner collects and writes as NDJSON traces to ``consensus/raw/`` in the build directory. The ``cft_model_test`` test, built with ``-DCFT_MODEL_DRIVER=ON``, runs the same scenarios with ``cft_model_driver``, which links the driver with the replayer and replays each trace in the process which records it; see :ccf_repo:`tests/cft_model/README.md`. + ``cft_model_driver --explore`` also changes the scenarios at random, and replays each changed run, to find runs in which the implementation and the model disagree, or which show an outcome that no scenario does. * **KV**: building with ``-DCCF_KV_TRACING=ON`` enables test-only trace capture in ``kv_test``. :ccf_repo:`tests/kv_trace_validation.py` captures the purpose-built ``KV trace *`` test cases and a seeded concurrent fuzzer, then checks every trace with ``kv-replay`` unless ``--capture-only`` is passed. Tracing is disabled in normal builds. diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index 02b9101c00e..7ce0ee23bcf 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -45,6 +45,7 @@ To report how many steps and traces show each outcome, and which outcomes no tra An outcome which no trace shows is a missing scenario, a step which the driver or the reduction cannot produce, or behaviour which the model allows and the implementation lacks. For example, the driver starts every node with an empty ledger, so no trace of a scenario shows `joinFromSnapshot`. +The [consensus model driver](../../../tests/cft_model/README.md#exploring) looks for runs which show such an outcome, by changing the scenarios at random. To compare with the implementation, configure the scenario build with `-DCOVERAGE=ON` and run the scenario test. [`coverage.sh`](../../../scripts/coverage.sh) then lists the lines of [`cft.h`](../../../src/consensus/aft/cft.h) which the scenarios do not execute: diff --git a/scripts/lean-format-checks.sh b/scripts/lean-format-checks.sh index 8da06ca49c2..b80ba58477c 100755 --- a/scripts/lean-format-checks.sh +++ b/scripts/lean-format-checks.sh @@ -57,6 +57,11 @@ for package in "${PACKAGES[@]}"; do lake --dir "$ROOT_DIR/tests/cft_scenarios" build export LEAN_PATH="$ROOT_DIR/tests/cft_scenarios/.lake/build/lib/lean" fi + # The modules of the model driver import one another. + if [ "$package" == "consensus" ]; then + lake --dir "$ROOT_DIR/tests/cft_model" build + export LEAN_PATH="$LEAN_PATH:$ROOT_DIR/tests/cft_model/.lake/build/lib/lean" + fi git -C "$ROOT_DIR" ls-files -z -- "${PATHSPEC[@]}" | while IFS= read -r -d '' file; do printf '%s\0' "$ROOT_DIR/$file" diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index feed49ad83d..3c7caea05b9 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -12,6 +12,7 @@ #include "tracing/fluentd_sink.h" +#include #include #include #include @@ -337,6 +338,11 @@ namespace std::unique_ptr capture; + // The edge counters of the consensus implementation, if this file is + // compiled with -fsanitize-coverage=inline-8bit-counters + char* edges_begin = nullptr; + char* edges_end = nullptr; + // Exports the trace to the Fluentd collector at a host and port, if given, // and creates the driver void start(const std::vector& host_and_port) @@ -421,6 +427,28 @@ extern "C" }); } + // The instrumentation calls this before main, with its counters + void __sanitizer_cov_8bit_counters_init(char* start, char* stop) + { + edges_begin = start; + edges_end = stop; + } + + lean_obj_res cft_edges() + { + return io([]() { + const auto size = static_cast(edges_end - edges_begin); + auto* edges = lean_alloc_sarray(1, size, size); + std::copy(edges_begin, edges_end, lean_sarray_cptr(edges)); + return edges; + }); + } + + lean_obj_res cft_time_limit(uint64_t seconds) + { + return io([&]() { ::alarm(static_cast(seconds)); }); + } + lean_obj_res cft_start_node(b_lean_obj_arg node) { const auto id = text(node); diff --git a/tests/cft_model/ModelDriver.lean b/tests/cft_model/ModelDriver.lean index 28030617321..0ae8ef5f8d8 100644 --- a/tests/cft_model/ModelDriver.lean +++ b/tests/cft_model/ModelDriver.lean @@ -1,7 +1,6 @@ -- Copyright (c) Microsoft Corporation. All rights reserved. -- Licensed under the Apache 2.0 License. -import Consensus.Replay.Session -import Scenarios +import ModelDriver.Explore /-! Runs consensus scenarios with the Lean model of the protocol in the same process. @@ -17,16 +16,6 @@ open CCF.MultiNodeTransitionSystem (nodeState) namespace ModelDriver -private def fail {α : Type} (message : String) : IO α := - throw (IO.userError message) - -/-- Take the trace records emitted since the last call, numbered after the `taken` before. -/ -private def takeRecords (source : String) (taken : Nat) : IO (Array Record) := do - (← takeTrace).mapIdxM - fun index line => do - let value ← IO.ofExcept (Json.parse line) - return { file := source, line := taken + index + 1, value } - /-- Run a scenario with its trace kept in memory, then replay the trace against the model: whole, and one command at a time, which must agree. @@ -51,29 +40,6 @@ private structure Run where /-- The replay, once a command has started the first node. -/ session : Option Session := none -/-- The nodes which have started, by name and model identifier. -/ -private def started (session : Session) : List (String × Nat) := - session.header.declared.zipIdx.filter - fun (_, id) => (nodeState session.state id).any fun node => !decide node.isFresh - -/-- -Each undelivered message of the driver must be a distinct envelope in the network of the -model. The model also keeps the messages which the driver has dropped. --/ -private def checkPending (session : Session) : IO Unit := do - let nodes := started session - for (source, sourceId) in nodes do - for (target, targetId) in nodes do - if sourceId != targetId then - let mut queued := - session.state.network.filter - fun envelope => envelope.source == sourceId && envelope.target == targetId - for pending in ← pendingMessages source target do - let packet ← IO.ofExcept (Json.parse pending) - let some envelope := queued.find? (packetMatches packet ·.payload) - | fail s!"the model holds no message from {source} to {target} like {pending}" - queued := queued.erase envelope - /-- Deliver the message which the model holds as `envelope`, wherever it is among the undelivered messages of the driver. @@ -150,25 +116,72 @@ private def runInStep : IO Result := do /-- The argument which selects the run in step, where a scenario is named. -/ private def inStep : String := "--in-step" +/-- Start this program again with `args`. Give its exit status and what it wrote to stderr. -/ +private def again (args : Array String) : IO (UInt32 × String) := do + let child ← + IO.Process.spawn + { cmd := (← IO.appPath).toString, args, stdout := .null, stderr := .piped } + let errors ← child.stderr.readToEnd + return (← child.wait, errors) + +/-- The status and the outcomes in the result file of a run, and its message. -/ +private def reported (file : System.FilePath) : IO (String × Json × String) := do + let result ← IO.ofExcept (Json.parse (← IO.FS.readFile file)) + return ( + ← IO.ofExcept (string result "status"), + ← IO.ofExcept (field result "outcomes"), + ← IO.ofExcept (string result "message") + ) + +/-- +Record the commands of a scenario, and run them as a program, which must show the outcomes +which the scenario shows. Give what went wrong, if anything did. +-/ +private def roundTrip (directory : System.FilePath) (name : String) : IO String := do + let program := directory / s!"{name}.program" + let (status, errors) ← again #["--record", name, program.toString] + if status != 0 then + return s!"recording its commands: {errors}" + let result := directory / s!"{name}.again.json" + let (status, errors) ← again #["--run", program.toString, result.toString] + if status != 0 then + return s!"running its commands: {errors}" + let (_, recorded, _) ← reported (program.withExtension "json") + let (status, ran, message) ← reported result + if status != "ok" then + return s!"its commands are {status}: {message}" + if recorded != ran then + return "its commands show other outcomes" + return "" + /-- Run every scenario and the run in step, each in a process of its own, because a process -can start the driver once. +can start the driver once. Also run each scenario from its commands, and explore briefly. -/ private def runAll : IO UInt32 := do - let app ← IO.appPath let names := (← scenarios.get).map (·.1) |>.reverse + let directory ← IO.FS.createTempDir let mut failed := 0 for name in names ++ [inStep] do - let child ← - IO.Process.spawn - { cmd := app.toString, args := #[name], stdout := .null, stderr := .piped } - let errors ← child.stderr.readToEnd - if (← child.wait) == 0 then + let (status, errors) ← again #[name] + let mut problem := if status == 0 then "" else errors + if problem.isEmpty && name != inStep then + problem ← roundTrip directory name + if problem.isEmpty then IO.println s!"{name}: ok" else failed := failed + 1 - IO.println s!"{name}: FAILED\n{errors}" - IO.println s!"{names.length + 1} runs, {failed} failed" + IO.println s!"{name}: FAILED\n{problem}" + -- A short exploration must run, whatever it finds + let (status, errors) ← + again #["--explore", (directory / "explore").toString, "--runs", "40"] + if status == 0 then + IO.println "--explore: ok" + else + failed := failed + 1 + IO.println s!"--explore: FAILED\n{errors}" + IO.FS.removeDirAll directory + IO.println s!"{names.length + 2} runs, {failed} failed" return if failed == 0 then 0 else 1 /-- Report the result of one run. -/ @@ -191,23 +204,55 @@ private def report (name : String) (run : IO Result) : IO UInt32 := do end ModelDriver +open ModelDriver in +/-- The options of an exploration, after its directory. -/ +private def exploration (options : Exploration) : List String → Except String Exploration + | [] => pure options + | name :: value :: rest => + match name, value.toNat? with + | "--seed", some seed => exploration { options with seed } rest + | "--runs", some runs => exploration { options with runs } rest + | "--jobs", some jobs => exploration { options with jobs := max jobs 1 } rest + | _, _ => throw s!"unknown option '{name} {value}'" + | [name] => throw s!"option '{name}' has no value" + open ModelDriver in /-- Run the scenario named by the argument, `--in-step` for the run in step with the model, -or `--all` for every scenario and the run in step. +or `--all` for every scenario and the run in step. `--record` writes the commands of a +scenario as a program, `--run` runs a program, and `--explore` changes the scenarios at +random and runs them. -/ def main (args : List String) : IO UInt32 := do + let unknown (name : String) : IO UInt32 := do + IO.eprintln s!"Unknown scenario '{name}'" + return 2 match args with | ["--all"] => runAll + | ["--record", name, file] => + match (← scenarios.get).lookup name with + | some steps => recordScenario name steps file + | none => unknown name + | ["--run", program] => runProgram program none + | ["--run", program, result] => runProgram program (some result) + | ["--scenario", program, name, output] => writeScenario program name output + | "--explore" :: directory :: options => + match exploration { directory } options with + | .ok options => explore options ((← scenarios.get).map (·.1) |>.reverse) + | .error error => + IO.eprintln error + return 2 + | ["--shrink", program, output] => shrink program output none 8 + | ["--shrink", program, output, "--outcome", outcome] => + shrink program output (some outcome) 8 | [name] => if name == inStep then report "in step" runInStep else match (← scenarios.get).lookup name with | some steps => report name (runScenario name steps) - | none => - IO.eprintln s!"Unknown scenario '{name}'" - return 2 + | none => unknown name | _ => - IO.eprintln "usage: cft_model_driver SCENARIO | --in-step | --all" + IO.eprintln + "usage: cft_model_driver SCENARIO | --in-step | --all | --record SCENARIO PROGRAM | --run PROGRAM [RESULT] | --explore DIRECTORY [--seed N] [--runs N] [--jobs N] | --shrink PROGRAM OUTPUT [--outcome OUTCOME] | --scenario PROGRAM NAME OUTPUT" return 2 diff --git a/tests/cft_model/ModelDriver/Command.lean b/tests/cft_model/ModelDriver/Command.lean new file mode 100644 index 00000000000..77550a2f374 --- /dev/null +++ b/tests/cft_model/ModelDriver/Command.lean @@ -0,0 +1,323 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import Scenarios + +/-! +Driver commands as data. + +A trace records each command of a scenario as its name and arguments, separated by commas. +A `Command` is such a record, which can be issued again, so that a scenario can be changed +and run without being compiled. A `Step` of a program is a command, or a choice which is +made when the program runs, among what the state of the driver then allows. +-/ + +open CftDriver + +namespace ModelDriver + +/-- A driver command: its name and arguments, as a trace records them. -/ +structure Command where + name : String + args : List String +deriving BEq, Inhabited + +namespace Command + +/-- Read a command from the text which a trace records for it. -/ +def parse (text : String) : Command := + match text.splitOn "," with + | name :: args => { name, args } + | [] => { name := text, args := [] } + +/-- The text which a trace records for the command. -/ +def text (command : Command) : String := + ",".intercalate (command.name :: command.args) + +private def number (text : String) : IO UInt64 := + match text.toNat? with + | some value => pure value.toUInt64 + | none => throw (IO.userError s!"expected a number, got '{text}'") + +private def texts (items : List String) : List Text := + items.map Text.mk + +/-- The nodes which `swap_nodes` retires and trusts: the trace lists them after `out` and `in`. -/ +private def changes (items : List String) : List String × List String := + let (retire, trust) := (items.dropWhile (· == "out")).span (· != "in") + (retire, trust.drop 1) + +/-- Issue the command to the driver. -/ +def run (command : Command) : IO Unit := do + match command.name, command.args with + | "start_node", [node] => startNode node + | "trust_node", [term, node] => trustNode term node + | "trust_nodes", term :: nodes => trustNodes term (texts nodes) + | "swap_node", [term, retire, trust] => swapNode term retire trust + | "swap_nodes", term :: items => + let (retire, trust) := changes items + swapNodes term (texts retire) (texts trust) + | "cleanup_nodes", term :: nodes => cleanupNodes term (texts nodes) + | "nominate_successor", [node] => nominateSuccessor node + | "nodes", nodes => CftDriver.nodes (texts nodes) + | "create_new_node", [node] => createNewNode node + | "replicate_new_configuration", term :: nodes => + replicateNewConfiguration term (texts nodes) + | "replicate", term :: entries => replicate term (texts entries) + | "periodic_one", [node, ms] => periodicOne node (← number ms) + | "periodic_all", [ms] => periodicAll (← number ms) + | "connect", [first, second] => connect first second + | "disconnect", [first, second] => disconnect first second + | "disconnect_node", [node] => disconnectNode node + | "reconnect", [first, second] => reconnect first second + | "reconnect_node", [node] => reconnectNode node + | "dispatch_one", [node] => dispatchOne node + | "dispatch_single", [source, target] => dispatchSingle source target + | "dispatch_nth", [source, target, position] => + dispatchNth source target (← number position) + | "pending_messages", [source, target] => discard (pendingMessages source target) + | "dispatch_all_once", [] => dispatchAllOnce + | "dispatch_all", [] => dispatchAll + | "drop_pending", [node] => dropPending node + | "drop_pending_to", [source, target] => dropPendingTo source target + | "drop_nth", [source, target, position] => dropNth source target (← number position) + | "hold_nth", [source, target, position] => holdNth source target (← number position) + | "dispatch_held", [position] => dispatchHeld (← number position) + | "shuffle_one", [node] => shuffleOne node + | "shuffle_all", [] => shuffleAll + | "state_one", [node] => stateOne node + | "state_all", [] => stateAll + | "summarise_log", [node] => summariseLog node + | "summarise_logs_all", [] => summariseLogsAll + | "summarise_messages", [source, target] => summariseMessages source target + | "assert_state_sync", [] => assertStateSync + | "assert_commit_safety", [node] => assertCommitSafety node + | "assert_commit_idx", [node, idx] => assertCommitIdx node (← number idx) + | "assert_detail", node :: detail :: expected => + assertDetail node detail (",".intercalate expected) + | "assert_!detail", node :: detail :: unexpected => + assertNotDetail node detail (",".intercalate unexpected) + | "assert_config", node :: idx :: nodes => + assertConfig node (← number idx) (texts nodes) + | "assert_absent_config", [node, idx] => assertAbsentConfig node (← number idx) + | "assert_last_txid", [node, txid] => assertLastTxid node txid + | "loop_until_sync", [] => loopUntilSync + | _, _ => throw (IO.userError s!"unknown command '{command.text}'") + +/-- Whether the command checks or prints the state, and so changes nothing. -/ +def observes (command : Command) : Bool := + command.name.startsWith "assert_" + || [ + "state_one", + "state_all", + "summarise_log", + "summarise_logs_all", + "summarise_messages", + "pending_messages" + ].contains + command.name + +/-- Whether the command orders messages at random, so that a run cannot be repeated. -/ +def shuffles (command : Command) : Bool := + command.name.startsWith "shuffle_" + +/-- The nodes which the command creates. -/ +def creates (command : Command) : List String := + match command.name, command.args with + | "start_node", [node] => [node] + | "create_new_node", [node] => [node] + | "trust_node", [_, node] => [node] + | "swap_node", [_, _, node] => [node] + | "trust_nodes", _ :: nodes => nodes + | "replicate_new_configuration", _ :: nodes => nodes + | "nodes", nodes => nodes + | "swap_nodes", _ :: items => (changes items).2 + | _, _ => [] + +/-- The commands which the driver cannot run unless the node which they name first exists. -/ +private def namesNode : List String := + [ + "nominate_successor", + "periodic_one", + "dispatch_one", + "dispatch_single", + "dispatch_nth", + "pending_messages", + "drop_pending", + "drop_pending_to", + "drop_nth", + "hold_nth", + "state_one", + "summarise_log", + "summarise_messages", + "assert_commit_safety", + "assert_commit_idx", + "assert_detail", + "assert_!detail", + "assert_config", + "assert_absent_config", + "assert_last_txid" + ] + +/-- The nodes which must exist for the driver to run the command. -/ +def needs (command : Command) : List String := + if command.name == "cleanup_nodes" then + command.args.drop 1 + else if namesNode.contains command.name then + command.args.take 1 + else + [] + +/-- The commands whose first argument is the term of the primary which they address. -/ +private def namesTerm : List String := + [ + "trust_node", + "trust_nodes", + "swap_node", + "swap_nodes", + "cleanup_nodes", + "replicate_new_configuration", + "replicate" + ] + +/-- The command, addressed to the primary in the highest term if it names a term. -/ +def toLatest (command : Command) : Command := + match command.args with + | _ :: rest => + if namesTerm.contains command.name then + { command with args := "latest" :: rest } + else + command + | [] => command + +/-- A node, a term or a number, as a scenario writes it. -/ +private def value (text : String) : String := + if text.toNat?.isSome || text == "latest" then text else text.quote + +private def list (items : List String) : String := + "[" ++ ", ".intercalate items ++ "]" + +/-- The line of a scenario in Lean which issues the command. -/ +def scenarioLine (command : Command) : Except String String := + let call (function : String) (args : List String) : Except String String := + pure (" ".intercalate (function :: args)) + let values (function : String) : Except String String := + call function (command.args.map value) + match command.name, command.args with + | "start_node", [_] => values "startNode" + | "trust_node", [_, _] => values "trustNode" + | "trust_nodes", term :: nodes => call "trustNodes" [value term, list (nodes.map value)] + | "swap_node", [_, _, _] => values "swapNode" + | "swap_nodes", term :: items => + let (retire, trust) := changes items + call "swapNodes" [value term, list (retire.map value), list (trust.map value)] + | "cleanup_nodes", term :: nodes => + call "cleanupNodes" [value term, list (nodes.map value)] + | "nominate_successor", [_] => values "nominateSuccessor" + | "replicate", term :: entries => + let entries := + entries.map fun entry => if entry == "$" then "signature" else entry.quote + call "replicate" [value term, list entries] + | "periodic_one", [_, _] => values "periodicOne" + | "periodic_all", [_] => values "periodicAll" + | "connect", [_, _] => values "connect" + | "disconnect", [_, _] => values "disconnect" + | "disconnect_node", [_] => values "disconnectNode" + | "reconnect", [_, _] => values "reconnect" + | "reconnect_node", [_] => values "reconnectNode" + | "dispatch_one", [_] => values "dispatchOne" + | "dispatch_single", [_, _] => values "dispatchSingle" + | "dispatch_nth", [_, _, _] => values "dispatchNth" + | "dispatch_all_once", [] => values "dispatchAllOnce" + | "dispatch_all", [] => values "dispatchAll" + | "drop_pending", [_] => values "dropPending" + | "drop_pending_to", [_, _] => values "dropPendingTo" + | "drop_nth", [_, _, _] => values "dropNth" + | "hold_nth", [_, _, _] => values "holdNth" + | "dispatch_held", [_] => values "dispatchHeld" + | "assert_commit_idx", [_, _] => values "assertCommitIdx" + | "assert_detail", node :: detail :: expected => + call "assertDetail" [value node, detail.quote, value (",".intercalate expected)] + | "loop_until_sync", [] => values "loopUntilSync" + | _, _ => throw s!"a scenario cannot issue '{command.text}'" + +end Command + +/-- +The scenario `name` in Lean, which issues the commands, and then the checks. Each of +the `comments` is a line before it. +-/ +def scenarioText (name : String) (comments : List String) + (commands checks : Array Command) + : Except String String := do + let lines (commands : Array Command) : Except String (List String) := + commands.toList.mapM fun command => (" " ++ ·) <$> command.scenarioLine + let header := + [ + "-- Copyright (c) Microsoft Corporation. All rights reserved.", + "-- Licensed under the Apache 2.0 License.", + "import CftDriver", + "", + "open CftDriver", + "" + ] + let issued ← lines commands + let checked ← lines checks + let body := + comments.map ("-- " ++ ·) + ++ [s!"initialize scenario {name.quote} do"] + ++ issued + ++ (if checked.isEmpty then [] else "" :: checked) + return "\n".intercalate (header ++ body) ++ "\n" + +/-- A step of a program. -/ +inductive Step where + /-- A command which the program gives. -/ + | given (command : Command) + /-- + A choice of the `kind` of command, such as holding an undelivered message. `value` + selects among those which the state of the driver allows when the program reaches the + step, and the step does nothing if there is none. + -/ + | choice (kind : String) (value : Nat) +deriving Inhabited + +namespace Step + +/-- The line of a program file for the step. A choice starts with `@`. -/ +def text : Step → String + | .given command => command.text + | .choice kind value => s!"@{kind} {value}" + +/-- Read a line of a program file. -/ +def parse (line : String) : Except String Step := + if line.startsWith "@" then + match (line.drop 1).toString.splitOn " " with + | [kind, value] => + match value.toNat? with + | some value => pure (.choice kind value) + | none => throw s!"expected a number in '{line}'" + | _ => throw s!"expected a kind and a number in '{line}'" + else + pure (.given (Command.parse line)) + +end Step + +/-- The text of a program file: a step on each line, after comments which start with `#`. -/ +def programText (comments : List String) (steps : Array Step) : String := + String.join ((comments.map ("# " ++ ·) ++ steps.toList.map Step.text).map (· ++ "\n")) + +/-- Read a program file. -/ +def readProgram (file : System.FilePath) : IO (Array Step) := do + let lines := (← IO.FS.readFile file).splitOn "\n" + let steps ← + (lines.filter fun line => !line.isEmpty && !line.startsWith "#").mapM + fun line => IO.ofExcept (Step.parse line) + return steps.toArray + +/-- The comments of a program file. -/ +def readComments (file : System.FilePath) : IO (List String) := do + let lines := (← IO.FS.readFile file).splitOn "\n" + return (lines.filter (·.startsWith "#")).map + fun line => (line.drop (if line.startsWith "# " then 2 else 1)).toString + +end ModelDriver diff --git a/tests/cft_model/ModelDriver/Episode.lean b/tests/cft_model/ModelDriver/Episode.lean new file mode 100644 index 00000000000..515c112a609 --- /dev/null +++ b/tests/cft_model/ModelDriver/Episode.lean @@ -0,0 +1,451 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import Consensus.Replay.Session +import ModelDriver.Command + +/-! +One run of the driver, checked against the model. + +A process can start the driver once, so each run has a process of its own. It reports +through files to the process which started it: what the model made of the run, the +commands which it issued, and the edges of the consensus implementation which it reached. +-/ + +open Lean +open CftDriver +open Consensus Consensus.Replay +open CCF.MultiNodeTransitionSystem (nodeState) + +namespace ModelDriver + +/-- Fail with `message`. -/ +def fail {α : Type} (message : String) : IO α := + throw (IO.userError message) + +/-- +The counters of the edges of the consensus implementation, each nonzero if the process has +run its edge. Empty unless the driver was compiled to count them. +-/ +@[extern "cft_edges"] +opaque edges : IO ByteArray + +/-- End the process if it is still running after `seconds`. -/ +@[extern "cft_time_limit"] +opaque timeLimit (seconds : UInt64) : IO Unit + +/-- Take the trace records emitted since the last call, numbered after the `taken` before. -/ +def takeRecords (source : String) (taken : Nat) : IO (Array Record) := do + (← takeTrace).mapIdxM + fun index line => do + let value ← IO.ofExcept (Json.parse line) + return { file := source, line := taken + index + 1, value } + +/-- The nodes which have started, by name and model identifier. -/ +def started (session : Session) : List (String × Nat) := + session.header.declared.zipIdx.filter + fun (_, id) => (nodeState session.state id).any fun node => !decide node.isFresh + +/-- +Each message which the driver has not delivered, or holds, must be a distinct envelope in +the network of the model. The model also keeps the messages which the driver has dropped. +-/ +def checkPending (session : Session) : IO Unit := do + let mut queued := session.state.network + for text in (← undeliveredMessages) ++ (← heldMessages) do + let message ← IO.ofExcept (Json.parse text) + let found : Except String (Option Model.Envelope) := do + let source ← session.header.nodeId (← string message "from") + let target ← session.header.nodeId (← string message "to") + let packet ← field message "packet" + return queued.find? + fun envelope => + envelope.source == source + && envelope.target == target + && packetMatches packet envelope.payload + match found with + | .ok (some envelope) => queued := queued.erase envelope + | .ok none => fail s!"the model holds no message like {text}" + | .error error => fail s!"{error} in {text}" + +/-- Why the driver stopped before the end of a run. -/ +structure Stop where + /-- `stuck`, `invariant` or `error`. -/ + kind : String + message : String + /-- The command which it could not run. -/ + command : String + +/-- What a run did, as the model sees it. -/ +structure Report where + /-- + `ok`, or `finding` if the driver and the model disagree or the driver fails, or `invalid` + if the model does not cover the run: it does not start a network, or it has a primary + append what no application would, such as the removal of a node which has not retired. + -/ + status : String := "ok" + /-- What kind of finding: `model`, `undelivered`, `invariant` or `error`. -/ + kind : String := "" + message : String := "" + /-- The command at which it was found. -/ + command : String := "" + /-- + The outcomes which the model shows, in order, before the driver confirms that the + network can make progress. + -/ + outcomes : Array String := #[] + /-- The outcomes which the model shows as the driver confirms that. -/ + settled : Array String := #[] + /-- + Commands which assert the state of each node, as the model has it, before the driver + confirms that. + -/ + checks : Array String := #[] + /-- Why the driver could not confirm that the network can make progress, if so. -/ + stuck : String := "" + +private def has (text : String) (parts : List String) : Bool := + parts.any fun part => (text.splitOn part).length > 1 + +/-- Whether the driver failed because the network cannot make progress. -/ +private def isStuck (error : String) : Bool := + has error ["Failed to reach state sync", "Can't currently loop until sync"] + +/-- +Why the driver stops when a command fails with `error`, or `none` if the command does not +apply in the state of the driver, which goes on. +-/ +private def stopFor (error : String) : Option String := + if has error + [ + "Found no primary in term", + "No undelivered message at position", + "No held message at position" + ] then + none + else if has error ["at unsafe commit idx"] then + some "invariant" + else if isStuck error then + some "stuck" + else + some "error" + +/-- Replay a run by command. An error is with the index of the command which fails. -/ +private def replayRun (records : Array Record) : Except (Nat × String) Session := do + let (_, declared) ← (prepareEvents records).mapError ((0, ·)) + let commands := splitCommands records + let some first := commands[0]? | throw (0, "the trace has no records") + let mut session ← (Session.start declared first).mapError ((0, ·)) + for index in [1:commands.size] do + session ← (session.feed commands[index]!).mapError ((index, ·)) + return session + +/-- The command which a record of a trace names, if it is the record of a command. -/ +private def commandText (record : Record) : Option String := + ((optional record.value "msg").bind (optional · "cmd")).bind (·.getStr?.toOption) + +/-- The text of the command at `index` in a trace. -/ +private def commandAt (records : Array Record) (index : Nat) : String := + (((splitCommands records)[index]?.bind (·[0]?)).bind commandText).getD "" + +/-- Commands which assert the leadership and the commit index of each node in the model. -/ +private def stateChecks (session : Session) : Array String := + (started session).toArray.flatMap + fun (node, id) => + match nodeState session.state id with + | none => #[] + | some state => + let leadership := + match state.role with + | .none => "None" + | .follower => "Follower" + | .preVoteCandidate => "PreVoteCandidate" + | .candidate => "Candidate" + | .primary => "Leader" + #[ + s!"assert_detail,{node},leadership_state,{leadership}", + s!"assert_commit_idx,{node},{state.commitIndex}" + ] + +/-- +Check what the driver has done against the model. Unless it has `stopped`, have it confirm +that the network can still make progress, and check that too. Also give the trace, and the +edges which the run reached before the driver confirmed progress. +-/ +private def conclude (stopped : Option Stop) + : IO (Report × Array Record × ByteArray) := do + let records ← takeRecords "run" 0 + let counters ← edges + -- A command which fails may leave a group of events unfinished. + let whole := + if stopped.any (·.kind == "error") then + (splitCommands records).pop.foldl (· ++ ·) #[] + else + records + match replayRun whole with + | .error (index, message) => + let command := commandAt whole index + -- The model only lets a primary append what an application would ask of it + let report : Report := + if index == 0 || has message ["disabled model action 'append'"] then + { status := "invalid", message, command } + else + { status := "finding", kind := "model", message, command } + return (report, records, counters) + | .ok session => + let outcomes := session.result.outcomes.map (·.name) + let checks := stateChecks session + let undelivered ← + try + checkPending session + pure none + catch error => + pure (some (toString error)) + if let some message := undelivered then + return ( + { status := "finding", kind := "undelivered", message, outcomes }, + records, + counters + ) + if let some stop := stopped then + let report : Report := + if stop.kind == "stuck" then + { outcomes, checks, stuck := stop.message } + else + { + status := "finding" + kind := stop.kind + message := stop.message + command := stop.command + outcomes + } + return (report, records, counters) + let failure ← + try + finish + pure "" + catch error => + pure (toString error) + let rest ← takeRecords "run" records.size + let records := records ++ rest + let report : Report := + if !failure.isEmpty && !isStuck failure then + { + status := "finding", kind := "error", message := failure, command := "finish" + } + else + match session.feed rest with + | .error message => + { status := "finding", kind := "model", message, command := "finish" } + | .ok settled => + { + stuck := failure + settled := + (settled.result.outcomes.extract outcomes.size).map (·.name) + } + return ({ report with outcomes, checks }, records, counters) + +/-- Write the result of a run to `file`, its edges beside it, and its trace if it is a finding. -/ +private def writeResult (file : System.FilePath) (report : Report) + (commands : Array String) (records : Array Record) (counters : ByteArray) + : IO Unit := do + let result := + Json.mkObj + [ + ("status", toJson report.status), + ("kind", toJson report.kind), + ("message", toJson report.message), + ("command", toJson report.command), + ("outcomes", toJson report.outcomes), + ("settled", toJson report.settled), + ("checks", toJson report.checks), + ("stuck", toJson report.stuck), + ("commands", toJson commands) + ] + IO.FS.writeBinFile (file.withExtension "edges") counters + if report.status == "finding" then + IO.FS.writeFile (file.withExtension "ndjson") + (String.join (records.toList.map (·.value.compress ++ "\n"))) + IO.FS.writeFile file result.compress + +/-- Print the status of a run, and give its exit status. -/ +private def announce (name : String) (report : Report) : IO UInt32 := do + if report.status == "ok" then + let status := + Json.mkObj + [ + ("status", toJson "ok"), + ("run", toJson name), + ("outcomes", toJson report.outcomes.size), + ("stuck", toJson report.stuck) + ] + IO.println status.compress + return 0 + IO.eprintln s!"{name}: {report.kind} at '{report.command}': {report.message}" + return 1 + +/-- +Run a scenario, and write the commands which it issued to `file`, as a program. Write +the result of the run beside it, as `runProgram` does, but with every edge which the +scenario reached. +-/ +def recordScenario (name : String) (steps : IO Unit) (file : System.FilePath) + : IO UInt32 := do + timeLimit 120 + startCapture + let stopped ← + try + steps + pure none + catch error => + pure (some { kind := "error", message := toString error, command := "" : Stop }) + let (report, records, _) ← conclude stopped + let commands := records.filterMap commandText + IO.FS.writeFile file (programText [name] (commands.map (.given <| Command.parse ·))) + writeResult (file.withExtension "json") report commands records (← edges) + announce name report + +/-- A message which the driver reports, with its position among those of its link. -/ +private structure Queued where + source : String + target : String + position : Nat + +private def queuedMessages (reported : Array String) : IO (Array Queued) := do + let mut messages : Array Queued := #[] + for text in reported do + let message ← IO.ofExcept (Json.parse text) + let source ← IO.ofExcept (string message "from") + let target ← IO.ofExcept (string message "to") + let before := + messages.filter fun other => other.source == source && other.target == target + messages := messages.push { source, target, position := before.size } + return messages + +/-- +The command for a choice, if the state of the driver allows one. `nodes` are those which +exist, and `isolated` those which a command has disconnected from every other. +-/ +private def choose (kind : String) (value : Nat) (nodes isolated : List String) + : IO (Option Command) := do + let pick (items : List String) : Option String := items[value % items.length]? + let named (name : String) (node : Option String) : Option Command := + node.map fun node => { name, args := [node] } + match kind with + | "hold" | "deliver" | "drop" => + let pending ← queuedMessages (← undeliveredMessages) + let some message := pending[value % pending.size]? | return none + let name := + if kind == "hold" then + "hold_nth" + else if kind == "deliver" then + "dispatch_nth" + else + "drop_nth" + return some + { name, args := [message.source, message.target, toString message.position] } + | "release" => + let held ← heldMessages + if held.isEmpty then + return none + return some { name := "dispatch_held", args := [toString (value % held.size)] } + | "tick" => + -- A node which is not primary always starts an election after 100 ms + let duration := if value / nodes.length % 3 == 0 then "10" else "100" + return (pick nodes).map + fun node => { name := "periodic_one", args := [node, duration] } + | "isolate" => return named "disconnect_node" (pick nodes) + | "rejoin" => return named "reconnect_node" (pick isolated) + | "nominate" => return named "nominate_successor" (pick nodes) + | "write" => + return some { name := "replicate", args := ["latest", s!"m{value % 1000}"] } + | "sign" => return some { name := "replicate", args := ["latest", "$"] } + | _ => fail s!"unknown choice '{kind}'" + +/-- +Run a program, and check the run against the model. A step which the state of the +driver does not allow is left out. Give the commands issued, with what `conclude` gives. +-/ +private def execute (steps : Array Step) + : IO (Report × Array Command × Array Record × ByteArray) := do + timeLimit 120 + startCapture + let mut issued : Array Command := #[] + let mut nodes : List String := [] + let mut isolated : List String := [] + let mut stopped : Option Stop := none + for step in steps do + let chosen ← + match step with + | .given command => pure (some command) + | .choice kind value => choose kind value nodes isolated + let some command := chosen | continue + -- The driver fails these, some after it has created a node + if command.needs.any (!nodes.contains ·) then + continue + if command.name == "start_node" && !nodes.isEmpty then + continue + if command.name != "replicate_new_configuration" + && command.creates.any (nodes.contains ·) then + continue + let created := command.creates.filter (!nodes.contains ·) + nodes := nodes ++ created + if command.name == "disconnect_node" then + isolated := command.args ++ isolated + if command.name == "reconnect_node" then + isolated := isolated.filter (!command.args.contains ·) + let failure ← + try + command.run + pure none + catch error => + pure (some (toString error)) + match failure with + | none => issued := issued.push command + | some message => + match stopFor message with + | none => + -- Left out, unless the driver created its nodes before it failed + unless created.isEmpty do + issued := issued.push command + | some kind => + issued := issued.push command + stopped := some { kind, message, command := command.text } + break + let (report, records, counters) ← conclude stopped + return (report, issued, records, counters) + +/-- +Run the program in `file`, and check the run against the model. Write to `result`, if +given: the status of the run, the outcomes which the model shows and the commands +issued, as JSON; the edges reached, beside it; and the trace, if the run is a finding. +-/ +def runProgram (file : System.FilePath) (result : Option System.FilePath) + : IO UInt32 := do + let (report, issued, records, counters) ← execute (← readProgram file) + match result with + | some result => + writeResult result report (issued.map (·.text)) records counters + return 0 + | none => announce file.toString report + +/-- +Run the program in `file`, which the model must accept, and write it to `output` as the +scenario `name` in Lean. The scenario issues the commands of the run, and asserts the +state in which they leave each node. +-/ +def writeScenario (file : System.FilePath) (name : String) (output : System.FilePath) + : IO UInt32 := do + let (report, issued, _, _) ← execute (← readProgram file) + if report.status != "ok" then + discard (announce file.toString report) + return 1 + let checks := report.checks.map Command.parse + match scenarioText name (← readComments file) issued checks with + | .ok text => + IO.FS.writeFile output text + return 0 + | .error error => + IO.eprintln s!"{file}: {error}" + return 1 + +end ModelDriver diff --git a/tests/cft_model/ModelDriver/Explore.lean b/tests/cft_model/ModelDriver/Explore.lean new file mode 100644 index 00000000000..f8d35cae16a --- /dev/null +++ b/tests/cft_model/ModelDriver/Explore.lean @@ -0,0 +1,631 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import ModelDriver.Episode + +/-! +Explores what the consensus implementation does when the scenarios are changed. + +The commands of each scenario are a seed. A program is changed at random, for example to +hold a message back until later, and run in a process of its own, which checks the run +against the model. A run is kept, to be changed in turn, if it makes the model show an +outcome, or reaches an edge of the implementation, which no run before it has. A run in +which the driver fails, or the driver and the model disagree, is a finding. +-/ + +open Lean +open Consensus.Replay (string field Outcome) + +namespace ModelDriver + +/-- How to explore. -/ +structure Exploration where + /-- Where the programs which are run, kept and found are written, with a report. -/ + directory : System.FilePath + /-- The changes made are a function of the seed. -/ + seed : Nat := 1 + /-- How many changed programs to run. -/ + runs : Nat := 1000 + /-- How many programs to run at once. -/ + jobs : Nat := 8 + +private abbrev Random := StateM StdGen + +/-- A number below `bound`. -/ +private def below (bound : Nat) : Random Nat := + modifyGet fun generator => randNat generator 0 (bound - 1) + +private def insert (steps : Array Step) (index : Nat) (step : Step) : Array Step := + steps.extract 0 index ++ #[step] ++ steps.extract index steps.size + +/-- Each kind as many times as its weight, so that a kind picked at random is that likely. -/ +private def weighted (kinds : List (String × Nat)) : Array String := + kinds.toArray.flatMap fun (kind, weight) => Array.replicate weight kind + +/-- The kinds of change, and how often each is made. -/ +private def changes : Array String := + weighted + [ + ("delay", 6), + ("tick", 3), + ("drop", 2), + ("deliver", 2), + ("partition", 2), + ("delete", 2), + ("burst", 2), + ("nominate", 1), + ("write", 1), + ("sign", 1), + ("duplicate", 1), + ("swap", 1), + ("latest", 1), + ("truncate", 1) + ] + +/-- The kinds of choice in a burst, and how often each is made. -/ +private def choices : Array String := + weighted + [ + ("tick", 2), + ("hold", 1), + ("release", 1), + ("deliver", 1), + ("drop", 1), + ("isolate", 1), + ("rejoin", 1), + ("nominate", 1), + ("write", 1), + ("sign", 1) + ] + +/-- +Change a program in one way, and describe the change, or give no description if the +program is as it was. The first step, which starts the network, stays. +-/ +private def change (steps : Array Step) : Random (Array Step × String) := do + let size := steps.size + let position := 1 + (← below size) + let value ← below 1000000 + let kind := changes[← below changes.size]! + match kind with + | "delay" | "partition" => + -- Hold a message, or disconnect a node, and undo that later + let later := position + (← below (size - position + 1)) + let (first, second) := + if kind == "delay" then ("hold", "release") else ("isolate", "rejoin") + let changed := + insert (insert steps later (.choice second value)) position (.choice first value) + return (changed, s!"{kind} {position}..{later}") + | "delete" => + if position < size then + return ( + steps.extract 0 position ++ steps.extract (position + 1) size, + s!"delete {position}" + ) + return (steps, "") + | "duplicate" => + match steps[position]? with + | some (.given command) => + if command.creates.isEmpty then + return (insert steps position (.given command), s!"duplicate {position}") + return (steps, "") + | _ => return (steps, "") + | "swap" => + match steps[position]?, steps[position + 1]? with + | some first, some second => + return ( + (steps.set! position second).set! (position + 1) first, + s!"swap {position}" + ) + | _, _ => return (steps, "") + | "latest" => + let changed := + steps.mapIdx + fun index step => + match step with + | .given command => if index < position then step else .given command.toLatest + | step => step + return (changed, s!"latest {position}") + | "burst" => + let count := 3 + (← below 6) + let mut burst := #[] + for _ in [0:count] do + burst := burst.push (.choice choices[← below choices.size]! (← below 1000000)) + return ( + steps.extract 0 position ++ burst ++ steps.extract position size, + s!"burst {position}+{count}" + ) + | "truncate" => + if position < size then + return (steps.extract 0 position, s!"truncate {position}") + return (steps, "") + | kind => return (insert steps position (.choice kind value), s!"{kind} {position}") + +/-- Why a program is run. -/ +private inductive Purpose where + /-- The commands of a scenario, which must run as they are. -/ + | seed + /-- A changed program. -/ + | explore + /-- The commands of a run which reached something new, which must do so again. -/ + | confirm + /-- The commands of the first run with a finding, to tell whether it happens again. -/ + | reproduce (signature : String) + +private structure Job where + /-- The scenario from which the program comes, and the changes made to it. -/ + origin : String + steps : Array Step + purpose : Purpose + +/-- A program which is changed to make others. -/ +private structure Entry where + origin : String + commands : Array Command +deriving Inhabited + +/-- A program changed in up to four ways. -/ +private def changed (corpus : Array Entry) : Random Job := do + let entry := corpus[← below corpus.size]! + let count := + match ← below 10 with + | 0 | 1 | 2 | 3 | 4 => 1 + | 5 | 6 | 7 => 2 + | 8 => 3 + | _ => 4 + let mut steps := entry.commands.map Step.given + let mut made := [] + for _ in [0:count] do + let (next, description) ← change steps + steps := next + unless description.isEmpty do + made := made ++ [description] + return { + origin := s!"{entry.origin} + {", ".intercalate made}", steps, purpose := .explore + } + +/-- What a run reported. -/ +private structure Ran where + /-- As `Report.status`, or `crash` if the run did not report. -/ + status : String + kind : String := "" + message : String := "" + command : String := "" + /-- The outcomes shown before the driver confirmed progress, each once. -/ + outcomes : List String := [] + /-- The outcomes shown as the driver confirmed progress, each once. -/ + settled : List String := [] + stuck : String := "" + commands : Array Command := #[] + edges : ByteArray := ByteArray.empty + /-- The file of its result, beside which its trace is, if it is a finding. -/ + result : System.FilePath + +private def strings (value : Json) (key : String) : Except String (List String) := do + let items ← (← field value key).getArr? + return (← items.mapM (·.getStr?)).toList + +/-- Read what a run wrote. `exit` is its exit status, and `errors` what it wrote to stderr. -/ +private def readRan (result : System.FilePath) (exit : UInt32) (errors : String) + : IO Ran := do + if exit != 0 || !(← result.pathExists) then + let last := ("\n".intercalate ((errors.splitOn "\n").reverse.take 12).reverse) + return { + status := "crash" + kind := "crash" + message := s!"exit status {exit}: {last}" + result + } + let value ← IO.ofExcept (Json.parse (← IO.FS.readFile result)) + let read : Except String Ran := do + return { + status := ← string value "status" + kind := ← string value "kind" + message := ← string value "message" + command := ← string value "command" + outcomes := (← strings value "outcomes").eraseDups + settled := (← strings value "settled").eraseDups + stuck := ← string value "stuck" + commands := ((← strings value "commands").map Command.parse).toArray + result + } + let ran ← IO.ofExcept read + return { ran with edges := ← IO.FS.readBinFile (result.withExtension "edges") } + +private abbrev quiet : IO.Process.StdioConfig := + { stdin := .inherit, stdout := .null, stderr := .piped } + +/-- Start this program again with `args`. -/ +private def start (args : Array String) : IO (IO.Process.Child quiet) := do + IO.Process.spawn { toStdioConfig := quiet, cmd := (← IO.appPath).toString, args } + +/-- Wait for runs which were started together, and read their results. -/ +private def finishRuns (started : Array (IO.Process.Child quiet × System.FilePath)) + : IO (Array Ran) := + started.mapM + fun (child, result) => do + let errors ← child.stderr.readToEnd + readRan result (← child.wait) errors + +/-- Run programs at once, each in a process of its own. -/ +private def runJobs (directory : System.FilePath) (jobs : Array Job) + : IO (Array Ran) := do + let mut started := #[] + for job in jobs, slot in [0:jobs.size] do + let program := directory / "run" / s!"{slot}.program" + let result := program.withExtension "json" + IO.FS.writeFile program (programText [job.origin] job.steps) + if ← result.pathExists then + IO.FS.removeFile result + started := + started.push (← start #["--run", program.toString, result.toString], result) + finishRuns started + +/-- A finding, as the first run which had it. -/ +private structure Finding where + kind : String + /-- Its kind and message, without what differs from run to run. -/ + signature : String + message : String + command : String + origin : String + /-- How many runs had it. -/ + count : Nat := 1 + /-- Whether its commands had it when they were run again. -/ + reproduced : Option Bool := none + +/-- A run which was kept: what it showed and reached which no run before it had. -/ +private structure Kept where + origin : String + outcomes : List String + edges : Nat + +private structure State where + corpus : Array Entry := #[] + /-- The outcomes which the scenarios show. -/ + shown : List String := [] + /-- The outcomes which a run has shown. -/ + outcomes : List String := [] + /-- The edges which a run has reached, each nonzero. -/ + edges : ByteArray := ByteArray.empty + /-- How many edges the scenarios reach. -/ + reached : Nat := 0 + kept : Array Kept := #[] + findings : Array Finding := #[] + /-- Why the driver could not confirm progress, and in how many runs. -/ + stuck : List (String × Nat) := [] + explored : Nat := 0 + ok : Nat := 0 + invalid : Nat := 0 + /-- Runs which reached something new, and did not when they were run again. -/ + unconfirmed : Nat := 0 + +private def byte (bytes : ByteArray) (index : Nat) : UInt8 := + if index < bytes.size then bytes.get! index else 0 + +/-- How many edges are nonzero. -/ +private def countEdges (edges : ByteArray) : Nat := + (List.range edges.size).countP fun index => edges.get! index != 0 + +/-- How many edges `reached` has which `known` lacks. -/ +private def freshEdges (known reached : ByteArray) : Nat := + (List.range reached.size).countP + fun index => reached.get! index != 0 && byte known index == 0 + +private def mergeEdges (known reached : ByteArray) : ByteArray := + (List.range reached.size).foldl + (fun merged index => + if reached.get! index != 0 then merged.set! index 1 else merged) + (known ++ ⟨Array.replicate (reached.size - known.size) 0⟩) + +/-- The text without what differs between runs of one kind: its numbers. -/ +private def signature (text : String) : String := + (text.foldl + (fun (collapsed, inNumber) character => + if character.isDigit then + (if inNumber then collapsed else collapsed.push 'N', true) + else + (collapsed.push character, false)) + ("", false)).1 + +private def Ran.signature (ran : Ran) : String := + s!"{ran.kind}: {ModelDriver.signature ran.message}" + +private def copyFile (source target : System.FilePath) : IO Unit := do + if ← source.pathExists then + IO.FS.writeBinFile target (← IO.FS.readBinFile source) + +/-- Take in the result of a run. Also give a program to run because of it, if any. -/ +private def absorb (directory : System.FilePath) (state : State) (job : Job) (ran : Ran) + : IO (State × Option Job) := do + let again (purpose : Purpose) : Job := + -- A run which did not report is run again as it was + let steps := if ran.commands.isEmpty then job.steps else ran.commands.map Step.given + { origin := job.origin, steps, purpose } + let newOutcomes := ran.outcomes.filter (!state.outcomes.contains ·) + let newEdges := freshEdges state.edges ran.edges + let keep (state : State) : IO State := do + let file := directory / "corpus" / s!"{state.corpus.size}.program" + let comments := [job.origin, s!"outcomes: {newOutcomes}", s!"edges: {newEdges}"] + IO.FS.writeFile file (programText comments (ran.commands.map Step.given)) + return { + state with + corpus := state.corpus.push { origin := job.origin, commands := ran.commands } + outcomes := state.outcomes ++ newOutcomes + edges := mergeEdges state.edges ran.edges + } + match job.purpose with + | .seed => + if ran.status == "ok" then + return (← keep state, none) + IO.println s!"{job.origin}: not explored, {ran.status}: {ran.message}" + return (state, none) + | .confirm => + if ran.status == "ok" && (!newOutcomes.isEmpty || newEdges > 0) then + let kept : Kept := + { origin := job.origin, outcomes := newOutcomes, edges := newEdges } + return ({ (← keep state) with kept := state.kept.push kept }, none) + return ({ state with unconfirmed := state.unconfirmed + 1 }, none) + | .reproduce signature => + let reproduced := ran.status != "ok" && ran.signature == signature + let findings := + state.findings.map + fun finding => + if finding.signature == signature then + { finding with reproduced := some reproduced } + else + finding + return ({ state with findings }, none) + | .explore => + let state := { state with explored := state.explored + 1 } + if ran.status == "ok" then + let stuck := + if ran.stuck.isEmpty then + state.stuck + else + let reason := ModelDriver.signature ran.stuck + match state.stuck.lookup reason with + | some count => + state.stuck.map + fun (other, runs) => + (other, if other == reason then count + 1 else runs) + | none => state.stuck ++ [(reason, 1)] + let state := { state with ok := state.ok + 1, stuck } + if newOutcomes.isEmpty && newEdges == 0 then + return (state, none) + return (state, some (again .confirm)) + if ran.status == "invalid" then + return ({ state with invalid := state.invalid + 1 }, none) + let signature := ran.signature + if state.findings.any (·.signature == signature) then + let findings := + state.findings.map + fun finding => + if finding.signature == signature then + { finding with count := finding.count + 1 } + else + finding + return ({ state with findings }, none) + let finding : Finding := + { + kind := ran.kind + signature + message := ran.message + command := ran.command + origin := job.origin + } + let follow := again (.reproduce signature) + let file := directory / "findings" / s!"{state.findings.size}.program" + let comments := [job.origin, s!"{ran.kind} at '{ran.command}'", ran.message] + IO.FS.writeFile file (programText (comments.map (·.replace "\n" " ")) follow.steps) + copyFile (ran.result.withExtension "ndjson") (file.withExtension "ndjson") + return ({ state with findings := state.findings.push finding }, some follow) + +private def cell (text : String) : String := + let line := (text.replace "\n" " ").replace "|" "/" + if line.length > 300 then (line.take 300).toString ++ "..." else line + +/-- What the exploration did, in Markdown. -/ +private def summary (options : Exploration) (state : State) (seconds : Nat) : String := + let found := state.outcomes.filter (!state.shown.contains ·) + let missing := (Outcome.all.map (·.name)).filter (!state.outcomes.contains ·) + let count (status : String) (runs : Nat) := s!"| {status} | {runs} |" + let lines := + [ + "# Exploration of the consensus scenarios", + "", + s!"Seed {options.seed}: {state.explored} changed programs run in {seconds} s.", + "", + "| Runs | |", + "| --- | ---: |", + count "Accepted by the model" state.ok, + count "Not covered by the model" state.invalid, + count "Findings" ((state.findings.map (·.count)).foldl (· + ·) 0), + count "Kept" state.kept.size, + count "New when first run, and not when run again" state.unconfirmed, + "", + s!"Outcomes: the scenarios show {state.shown.length} of {Outcome.all.length}, and the runs kept {found.length} more: {found}.", + s!"No run shows: {missing}.", + "", + s!"Edges: the scenarios reach {state.reached} of {state.edges.size}, and the runs kept {countEdges state.edges - state.reached} more.", + "", + "## Runs kept", + "", + "| Program | From | New outcomes | New edges |", + "| --- | --- | --- | ---: |" + ] + ++ (state.kept.toList.zipIdx (state.corpus.size - state.kept.size)).map + (fun (kept, index) => + s!"| corpus/{index}.program | {cell kept.origin} | {", ".intercalate kept.outcomes} | {kept.edges} |") + ++ [ + "", + "## Findings", + "", + "| Program | Kind | Runs | Again | From | At | Message |", + "| --- | --- | ---: | --- | --- | --- | --- |" + ] + ++ state.findings.toList.zipIdx.map + (fun (finding, index) => + let again := + match finding.reproduced with + | some true => "yes" + | some false => "no" + | none => "" + s!"| findings/{index}.program | {finding.kind} | {finding.count} | {again} | {cell finding.origin} | {cell finding.command} | {cell finding.message} |") + ++ [ + "", + "## Runs after which the driver could not confirm progress", + "", + "| Reason | Runs |", + "| --- | ---: |" + ] + ++ state.stuck.map fun (reason, runs) => s!"| {cell reason} | {runs} |" + "\n".intercalate lines ++ "\n" + +/-- The jobs in groups of at most `size`. -/ +private def groups (jobs : Array Job) (size : Nat) : List (Array Job) := + (List.range ((jobs.size + size - 1) / size)).map + fun group => jobs.extract (group * size) ((group + 1) * size) + +/-- +Make the program in `file` shorter: leave out steps for as long as its run shows `outcome` +and is accepted by the model or, if no outcome is given, is a finding of the same kind +and message, numbers aside. Write the program left to `output`. +-/ +def shrink (file output : System.FilePath) (outcome : Option String) (jobs : Nat) + : IO UInt32 := do + let directory ← IO.FS.createTempDir + IO.FS.createDirAll (directory / "run") + let run (programs : Array (Array Step)) : IO (Array Ran) := do + let mut ran := #[] + for group + in groups + (programs.map ({ origin := file.toString, steps := ·, purpose := .explore })) + jobs do + ran := ran ++ (← runJobs directory group) + return ran + let given ← readProgram file + let some first := (← run #[given])[0]? | return 1 + let keeps (ran : Ran) : Bool := + match outcome with + | some outcome => ran.status == "ok" && ran.outcomes.contains outcome + | none => + -- The same finding, and still before or after the driver confirms progress + ran.status != "ok" + && ran.status != "invalid" + && ran.status == first.status + && ran.signature == first.signature + && (ran.command == "finish") == (first.command == "finish") + unless keeps first do + IO.eprintln + s!"{file}: its run is {first.status}, and does not show what is to be kept: {first.message}" + IO.FS.removeDirAll directory + return 1 + let commands (ran : Ran) (steps : Array Step) : Array Step := + if ran.commands.isEmpty then steps else ran.commands.map Step.given + let mut steps := commands first given + let mut last := first + -- Leave out parts of `size` steps, after the first step, then smaller parts + let mut size := max 1 (steps.size / 2) + let mut rounds := 0 + while rounds < 1000 do + rounds := rounds + 1 + let parts := (steps.size - 1 + size - 1) / size + let candidates := + (List.range parts).toArray.map + fun part => + steps.extract 0 (1 + part * size) + ++ steps.extract (1 + (part + 1) * size) steps.size + let results ← run candidates + match (candidates.zip results).find? fun (_, ran) => keeps ran with + | some (candidate, ran) => + steps := commands ran candidate + last := ran + size := max 1 (min size (steps.size / 2)) + | none => + if size == 1 then + break + size := (size + 1) / 2 + IO.FS.removeDirAll directory + let kept := + match outcome with + | some outcome => [s!"shows {outcome}"] + | none => [s!"{last.kind} at '{last.command}'", last.message.replace "\n" " "] + IO.FS.writeFile output (programText (s!"{file}, shrunk" :: kept) steps) + IO.println s!"{given.size} steps shrunk to {steps.size}: {kept.head!}" + return 0 + +/-- +Record the commands of each scenario, change them at random and run the changed programs. +Write the programs kept and found, and a report, to the directory. +-/ +def explore (options : Exploration) (scenarios : List String) : IO UInt32 := do + let directory := options.directory + for part in ["run", "seeds", "corpus", "findings"] do + IO.FS.createDirAll (directory / part) + let began ← IO.monoMsNow + -- What the scenarios show and reach is not new + let mut state : State := {} + let mut seeds : Array Job := #[] + let recordings : Array Job := + scenarios.toArray.map fun name => { origin := name, steps := #[], purpose := .seed } + for jobs in groups recordings options.jobs do + let started ← + jobs.mapM + fun job => do + let program := directory / "seeds" / s!"{job.origin}.program" + return ( + ← start #["--record", job.origin, program.toString], + program.withExtension "json" + ) + for job in jobs, ran in (← finishRuns started) do + if ran.status != "ok" then + IO.println s!"{job.origin}: not explored, {ran.status}: {ran.message}" + else + -- Without the checks of the scenario, which a change would fail + let commands := + ran.commands.filter fun command => !command.observes && !command.shuffles + seeds := seeds.push { job with steps := commands.map Step.given } + state := + { + state with + outcomes := (state.outcomes ++ ran.outcomes ++ ran.settled).eraseDups + edges := mergeEdges state.edges ran.edges + } + state := { state with shown := state.outcomes, reached := countEdges state.edges } + for jobs in groups seeds options.jobs do + for job in jobs, ran in (← runJobs directory jobs) do + state := (← absorb directory state job ran).1 + if state.corpus.isEmpty then + IO.eprintln "no scenario can be explored" + return 1 + IO.println + s!"{state.corpus.size} scenarios show {state.shown.length} outcomes and reach {state.reached} of {state.edges.size} edges" + -- Change, run, and run again what is new + let mut generator := mkStdGen options.seed + let mut waiting : Array Job := #[] + let mut planned := 0 + let mut batches := 0 + while planned < options.runs || !waiting.isEmpty do + let mut jobs := waiting.extract 0 options.jobs + waiting := waiting.extract options.jobs waiting.size + while jobs.size < options.jobs && planned < options.runs do + let (job, next) := (changed state.corpus).run generator + generator := next + jobs := jobs.push job + planned := planned + 1 + for job in jobs, ran in (← runJobs directory jobs) do + let (next, follow) ← absorb directory state job ran + state := next + if let some follow := follow then + waiting := waiting.push follow + batches := batches + 1 + if batches % 50 == 0 then + IO.println + s!"{state.explored} runs: {state.kept.size} kept, {state.outcomes.length} outcomes, {countEdges state.edges} edges, {state.findings.size} findings" + let seconds := ((← IO.monoMsNow) - began) / 1000 + let report := summary options state seconds + IO.FS.writeFile (directory / "report.md") report + IO.println report + return 0 + +end ModelDriver diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index a20eec9e338..56cc7c74d32 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -16,10 +16,59 @@ build/cft_model_driver --all It replays each command as soon as the driver has run it, and requires each undelivered message of the driver to be a message which the model holds. It delivers the messages which the model holds, newest first, and asserts of the driver the commit indexes of the model. - `--all` runs every scenario and the run in step, each in a process of its own, because a process can start the driver once. + It also runs each scenario again from its [recorded commands](#programs), which must show the same outcomes, and [explores](#exploring) briefly. A run prints its status as JSON. It exits with status 1 if a command fails or the model rejects the trace, and 2 if its arguments are wrong. +## Programs + +A trace records each command of a scenario as its name and arguments, separated by commas. +A program is a file of such commands, one on each line, which the model driver runs without compiling a scenario: + +```sh +build/cft_model_driver --record .program +build/cft_model_driver --run .program [.json] +``` + +- `--record` runs a scenario and writes the commands which it issued as a program. +- `--run` runs a program, and checks its trace against the model: before the driver confirms that the network can make progress, and after. + With a result file, it writes there the status of the run, the [outcomes](../../lean/consensus/replay/README.md#outcomes) which the model shows, and the commands which it issued. + It writes beside it the edges of the consensus implementation which the run reached, and the trace if the run is a finding. + +A line of a program which starts with `#` is a comment. +One which starts with `@` is a choice, which the driver makes when the run reaches it, among what its state then allows. +For example, `@hold 7` holds one of the undelivered messages, and `@release 7` delivers one of those held. +The number selects which, so a program always runs in the same way. +A step which the state does not allow is left out, such as a command for a node which does not exist. + +## Exploring + +```sh +build/cft_model_driver --explore [--seed N] [--runs N] [--jobs N] +``` + +The commands of each scenario, without its assertions, are a first program. +The model driver changes a program at random: it holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, or leaves out, repeats or swaps commands. +It runs each changed program in a process of its own. + +- A run is kept, to be changed in turn, if the model shows an outcome, or the run reaches an edge of [`cft.h`](../../src/consensus/aft/cft.h), which no run before it has. + It is run a second time first, and is only kept if it does so again. +- A run is a finding if the driver fails, for example because a commit is unsafe, or if the model rejects its trace. + Findings with the same message, numbers aside, are counted together. +- A run is neither if the model does not cover it, for example because a primary is asked to remove a node which has not retired. + +The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, and `report.md`. +The same seed gives the same changes. + +```sh +build/cft_model_driver --shrink [--outcome ] +build/cft_model_driver --scenario .lean +``` + +- `--shrink` leaves out steps of a program for as long as its run is the same finding or, with `--outcome`, still shows that outcome. +- `--scenario` writes a program as a scenario in Lean, which ends by asserting the leadership and the commit index of each node. + ## Building The model driver needs the same prerequisites as [`cft_driver`](../cft_scenarios/README.md#running-scenarios), and CMake builds it only on request: @@ -38,3 +87,8 @@ For the model driver, `lake` builds the consensus model, and fetches from GitHub [`ModelDriver.lean`](ModelDriver.lean) imports the scenarios and [`Session.lean`](../../lean/consensus/Consensus/Replay/Session.lean), and defines `main`. `lake` compiles it, the scenarios and the model to static libraries. `cft_model_driver` links them with [`driver.cpp`](../../src/consensus/aft/test/driver.cpp) and the Lean runtime, as `cft_driver` does the scenarios. + +- [`Command.lean`](ModelDriver/Command.lean): driver commands as data, programs, and scenarios written from them. +- [`Episode.lean`](ModelDriver/Episode.lean): one run of the driver in a process, checked against the model. +- [`Explore.lean`](ModelDriver/Explore.lean): the changes made to programs, the runs kept, and the findings. +- [`edges.allowlist`](edges.allowlist): the functions whose edges the driver counts, for which `driver.cpp` is compiled with `-fsanitize-coverage`. diff --git a/tests/cft_model/edges.allowlist b/tests/cft_model/edges.allowlist new file mode 100644 index 00000000000..03088f60913 --- /dev/null +++ b/tests/cft_model/edges.allowlist @@ -0,0 +1,8 @@ +# The functions whose edges cft_model_driver counts: the members of aft::Aft, +# the consensus implementation in src/consensus/aft/cft.h, and the lambdas in +# them. +src:* +fun:_ZN3aft3AftI* +fun:_ZNK3aft3AftI* +fun:_ZZN3aft3AftI* +fun:_ZZNK3aft3AftI* From c3ff11ec7ab85449bd34b799aeb649eb71e36ce7 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 10:16:32 +0000 Subject: [PATCH 10/31] Add consensus scenarios found by exploration Seven scenarios, each of which makes the model show an outcome which no scenario showed: an acknowledgement, a vote and a request for a vote from a term which their recipient has left, the answer to a request for a pre-vote which reaches a candidate, a vote refused only because the primary of the term is known, a nomination of a node which has no signed configuration, and an election won by the only member of a configuration. cft_model_driver --explore found a run for each, --shrink left out the steps which the outcome does not need, and --scenario wrote the rest in Lean, with assertions of the state in which it ends. Two were then written to end with one primary, so that every run of them gives the same trace. The scenarios now show 76 of the 79 outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/cft_scenarios/Scenarios.lean | 7 +++ .../Scenarios/nominate_new_node.lean | 19 ++++++++ .../pre_vote_response_to_candidate.lean | 38 ++++++++++++++++ .../Scenarios/sole_member_election.lean | 43 ++++++++++++++++++ .../Scenarios/stale_ack_after_reelection.lean | 34 ++++++++++++++ .../Scenarios/stale_vote_after_new_term.lean | 45 +++++++++++++++++++ .../Scenarios/stale_vote_request.lean | 38 ++++++++++++++++ .../vote_request_after_append_entries.lean | 37 +++++++++++++++ 8 files changed, 261 insertions(+) create mode 100644 tests/cft_scenarios/Scenarios/nominate_new_node.lean create mode 100644 tests/cft_scenarios/Scenarios/pre_vote_response_to_candidate.lean create mode 100644 tests/cft_scenarios/Scenarios/sole_member_election.lean create mode 100644 tests/cft_scenarios/Scenarios/stale_ack_after_reelection.lean create mode 100644 tests/cft_scenarios/Scenarios/stale_vote_after_new_term.lean create mode 100644 tests/cft_scenarios/Scenarios/stale_vote_request.lean create mode 100644 tests/cft_scenarios/Scenarios/vote_request_after_append_entries.lean diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 387f0325e90..f007e1ae50c 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -26,10 +26,12 @@ import Scenarios.liveness_through_conflict import Scenarios.matching_partial import Scenarios.message_type_summaries import Scenarios.multi_election +import Scenarios.nominate_new_node import Scenarios.nominate_successor import Scenarios.nominate_without_successor import Scenarios.pre_vote import Scenarios.pre_vote_double_leader +import Scenarios.pre_vote_response_to_candidate import Scenarios.pre_vote_split import Scenarios.reconfig_01_12 import Scenarios.reconfig_01_23 @@ -55,12 +57,17 @@ import Scenarios.retirement_nominates_most_replicated import Scenarios.retirement_rollback import Scenarios.rollback_safety import Scenarios.soft_rollback +import Scenarios.sole_member_election +import Scenarios.stale_ack_after_reelection +import Scenarios.stale_vote_after_new_term +import Scenarios.stale_vote_request import Scenarios.startup import Scenarios.startup_2nodes import Scenarios.suffix_collision_1 import Scenarios.suffix_collision_2 import Scenarios.suffix_collision_3 import Scenarios.swap_single_node +import Scenarios.vote_request_after_append_entries /-! Every scenario. Importing this module registers them, for `Main` and for other programs diff --git a/tests/cft_scenarios/Scenarios/nominate_new_node.lean b/tests/cft_scenarios/Scenarios/nominate_new_node.lean new file mode 100644 index 00000000000..44576647e69 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/nominate_new_node.lean @@ -0,0 +1,19 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node which has no signed configuration ignores a nomination. The primary +-- adds node 2 and nominates it at once, before node 2 has any entry. +-- Found by `cft_model_driver --explore`, from `pre_vote_split`. +initialize scenario "nominate_new_node" do + startNode 0 + swapNodes 2 [] [2] + nominateSuccessor 0 + dispatchAll + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 0 + assertDetail 2 "leadership_state" "Follower" + assertCommitIdx 2 0 diff --git a/tests/cft_scenarios/Scenarios/pre_vote_response_to_candidate.lean b/tests/cft_scenarios/Scenarios/pre_vote_response_to_candidate.lean new file mode 100644 index 00000000000..08ce4b99c1a --- /dev/null +++ b/tests/cft_scenarios/Scenarios/pre_vote_response_to_candidate.lean @@ -0,0 +1,38 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A candidate ignores the answer to its request for a pre-vote. Nodes 1 and 2 +-- campaign at once, and each becomes a candidate in term 3 on the pre-vote of +-- the other. Node 0 votes for node 1 in term 3 before it answers the request +-- for a pre-vote of node 2, so its answer reaches a candidate of that term. +-- Found by `cft_model_driver --explore`, from `replicate_on_latest_leader`, +-- then written to end with one primary. +initialize scenario "pre_vote_response_to_candidate" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 1 100 + periodicOne 2 100 + dispatchSingle 1 2 + dispatchSingle 2 1 + dispatchSingle 1 2 + dispatchSingle 2 1 + dispatchSingle 1 0 + dispatchSingle 1 0 + dispatchSingle 2 0 + dispatchOne 0 + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Leader" + assertCommitIdx 1 3 + assertDetail 2 "leadership_state" "Candidate" + assertCommitIdx 2 3 diff --git a/tests/cft_scenarios/Scenarios/sole_member_election.lean b/tests/cft_scenarios/Scenarios/sole_member_election.lean new file mode 100644 index 00000000000..a24bbe7ec2c --- /dev/null +++ b/tests/cft_scenarios/Scenarios/sole_member_election.lean @@ -0,0 +1,43 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node which is the only member of its configuration wins an election +-- without asking for a vote. The primary, node 2, retires itself and node 0. +-- Node 1 learns that the configuration which leaves it alone is committed, +-- times out, and becomes primary at once. +-- Found by `cft_model_driver --explore`, from +-- `retired_committed_delayed_nomination`. +initialize scenario "sole_member_election" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 0 10 + dispatchSingle 0 2 + periodicOne 2 100 + dispatchSingle 2 1 + dispatchSingle 1 2 + dispatchSingle 2 1 + dispatchSingle 1 2 + dispatchSingle 2 1 + dispatchSingle 1 2 + swapNodes 3 [0, 2] [] + replicate 3 [signature] + periodicOne 2 10 + dispatchSingle 2 1 + dispatchSingle 1 2 + periodicOne 2 10 + dispatchSingle 2 1 + periodicOne 1 100 + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Leader" + assertCommitIdx 1 5 + assertDetail 2 "leadership_state" "Leader" + assertCommitIdx 2 5 diff --git a/tests/cft_scenarios/Scenarios/stale_ack_after_reelection.lean b/tests/cft_scenarios/Scenarios/stale_ack_after_reelection.lean new file mode 100644 index 00000000000..54e71bc3f35 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/stale_ack_after_reelection.lean @@ -0,0 +1,34 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A primary ignores an acknowledgement from an earlier term in which it was +-- also primary. Node 4 is elected in term 3, and the acknowledgement of node 1 +-- is held. Node 0 is elected in term 4, and node 4 again in term 5. Only then +-- does the acknowledgement arrive. +-- Found by `cft_model_driver --explore`, from `fancy_election_2`. +initialize scenario "stale_ack_after_reelection" do + startNode 0 + trustNode 2 1 + trustNode 2 4 + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 4 110 + dispatchAll + holdNth 1 4 0 + periodicOne 0 110 + dispatchAll + periodicOne 4 110 + dispatchAll + dispatchHeld 0 + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 4 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 4 + assertDetail 4 "leadership_state" "Leader" + assertCommitIdx 4 4 diff --git a/tests/cft_scenarios/Scenarios/stale_vote_after_new_term.lean b/tests/cft_scenarios/Scenarios/stale_vote_after_new_term.lean new file mode 100644 index 00000000000..d21cb445101 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/stale_vote_after_new_term.lean @@ -0,0 +1,45 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node ignores a vote for an election which it has since lost. Node 2 +-- campaigns in term 3, and the vote of node 4 is held. While node 2 is +-- disconnected, node 1 is elected in term 4. Node 2 hears of that term, and +-- only then does the vote arrive. +-- Found by `cft_model_driver --explore`, from `soft_rollback`. +initialize scenario "stale_vote_after_new_term" do + startNode 0 + trustNode 2 1 + trustNodes 2 [2, 3, 4] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 2 110 + dispatchOne 2 + dispatchSingle 3 2 + dispatchSingle 4 2 + dispatchOne 2 + disconnectNode 2 + holdNth 4 2 0 + periodicOne 1 110 + dispatchAll + reconnectNode 2 + periodicAll 10 + dispatchAll + dispatchHeld 0 + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 4 + assertDetail 1 "leadership_state" "Leader" + assertCommitIdx 1 4 + assertDetail 2 "leadership_state" "Follower" + assertCommitIdx 2 4 + assertDetail 3 "leadership_state" "Follower" + assertCommitIdx 3 4 + assertDetail 4 "leadership_state" "Follower" + assertCommitIdx 4 4 diff --git a/tests/cft_scenarios/Scenarios/stale_vote_request.lean b/tests/cft_scenarios/Scenarios/stale_vote_request.lean new file mode 100644 index 00000000000..7b7dfb01f17 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/stale_vote_request.lean @@ -0,0 +1,38 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node refuses a vote in a term which it has left. Node 1 is elected in +-- term 3 with the vote of node 0, and its requests to node 2 are delayed. +-- Node 0 then campaigns, and node 2 hears of term 4 from it before the +-- request of node 1 for a vote in term 3 arrives. +-- Found by `cft_model_driver --explore`, from `indirect_commit_stale_leader`, +-- then written to end with one primary. +initialize scenario "stale_vote_request" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 1 110 + dispatchSingle 1 0 + dispatchSingle 0 1 + dispatchSingle 1 0 + dispatchSingle 0 1 + periodicOne 0 110 + dispatchSingle 0 2 + dispatchSingle 2 0 + dispatchSingle 0 2 + dispatchSingle 1 2 + dispatchSingle 1 2 + dispatchAll + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 3 + assertDetail 2 "leadership_state" "Follower" + assertCommitIdx 2 3 diff --git a/tests/cft_scenarios/Scenarios/vote_request_after_append_entries.lean b/tests/cft_scenarios/Scenarios/vote_request_after_append_entries.lean new file mode 100644 index 00000000000..522f22ba009 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/vote_request_after_append_entries.lean @@ -0,0 +1,37 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node refuses a vote only because it knows the primary of the term. Node 2 +-- is elected in term 3 with the vote of node 1. Its first AppendEntries +-- overtakes its requests to node 0, which learns of the term and its primary, +-- and then refuses the vote without having voted for another node. +-- Found by `cft_model_driver --explore`, from +-- `retired_committed_delayed_nomination`. +initialize scenario "vote_request_after_append_entries" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicOne 0 10 + dispatchSingle 0 2 + dispatchSingle 2 0 + periodicOne 2 100 + dispatchSingle 2 1 + dispatchSingle 1 2 + dispatchSingle 2 1 + dispatchSingle 1 2 + dispatchNth 2 0 2 + dispatchSingle 2 0 + dispatchSingle 2 0 + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 0 + assertDetail 2 "leadership_state" "Leader" + assertCommitIdx 2 3 From c94ae7f051816cad2a4e92b87e2ebd4bde74a49a Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 11:58:23 +0000 Subject: [PATCH 11/31] Start the election timer of scenario nodes as real nodes do The driver started the election timer of each node as it created the node. A real node starts it when it first holds a signature while it is in its latest configuration. A scenario could therefore have a node campaign which no real node would, and the model rejects such a trace. An exploration of 20,000 changed scenarios did so in 125 of them. No scenario changes: 65 of the 66 traces are identical, and that of nominate_successor varies between runs as it did before. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.h | 5 ++++- tests/cft_scenarios/README.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 54516f37dbf..22818128117 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -278,7 +278,8 @@ class CftDriver { cft->init_as_backup(0, 0, {}); } - cft->start_ticking(); + // The node starts its election timer as a real node does: when it first + // holds a signature while it is in its latest configuration. if (_nodes.find(node_id) != _nodes.end()) { @@ -306,6 +307,8 @@ class CftDriver for (auto& node : _nodes) { node.second.cft->add_configuration(0, configuration); + // No node can sign before one is elected + node.second.cft->start_ticking(); } } diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 340a91e2ea7..a7b5a619050 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -33,6 +33,8 @@ Scenarios are ordinary Lean, so they can also use loops, conditions and definiti After each command which can alter state, the driver asserts that the commit index of every node is safe. After the last command, it confirms that the network can still make progress. +A node starts its election timer as a real node does, when it first holds a signature while it is in its latest configuration. +Advancing its clock before that starts no election. ## Adding a scenario From e7ed96248f9de1b75c884665e37aa8287cba1d43 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 11:58:24 +0000 Subject: [PATCH 12/31] Add a scenario which rolls back a retirement signature The signature which follows the removal of a node is rolled back on that node, and the removal is kept, so its retirement goes from Signed back to Ordered. No scenario did this, and exploration did not find it in 80,000 runs: it needs a primary which keeps an unsigned removal, and so an election before anything has committed. The scenarios now show 77 of the 79 outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/cft_scenarios/Scenarios.lean | 1 + .../retirement_signature_rollback.lean | 63 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/cft_scenarios/Scenarios/retirement_signature_rollback.lean diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index f007e1ae50c..f57a3e7fd16 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -55,6 +55,7 @@ import Scenarios.retired_node_delayed_responses import Scenarios.retirement_commit_below_signature import Scenarios.retirement_nominates_most_replicated import Scenarios.retirement_rollback +import Scenarios.retirement_signature_rollback import Scenarios.rollback_safety import Scenarios.soft_rollback import Scenarios.sole_member_election diff --git a/tests/cft_scenarios/Scenarios/retirement_signature_rollback.lean b/tests/cft_scenarios/Scenarios/retirement_signature_rollback.lean new file mode 100644 index 00000000000..7ad35c1e09d --- /dev/null +++ b/tests/cft_scenarios/Scenarios/retirement_signature_rollback.lean @@ -0,0 +1,63 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- The signature which follows the removal of a node is rolled back on that +-- node, and the removal is not. +-- No acknowledgement is delivered before the end, so nothing commits, and a +-- node which becomes primary keeps the entries after its last signature. +initialize scenario "retirement_signature_rollback" do + startNode 0 + trustNodes 2 [1, 2, 3, 4] + replicate 2 [signature] + dispatchAll + periodicOne 0 10 + dispatchOne 0 + -- Initial network state [0, 1, 2, 3, 4], with nothing committed + + -- 1 is elected in term 3, with the votes of 0 and 2 + periodicOne 1 100 + dispatchOne 1 + dispatchSingle 0 1 + dispatchSingle 2 1 + dispatchOne 1 + dispatchSingle 0 1 + dispatchSingle 2 1 + assertDetail 1 "leadership_state" "Leader" + + -- 1 removes 4 at index 4, which every node receives + swapNodes 3 [4] [] + periodicOne 1 10 + dispatchOne 1 + assertDetail 4 "retirement_phase" "Ordered" + + -- 1 signs at index 5, and only 4 receives the signature + replicate 3 [signature] + periodicOne 1 10 + dispatchSingle 1 4 + assertDetail 4 "retirement_phase" "Signed" + assertLastTxid 4 "3.5" + + -- 2 holds the removal without its signature. It is elected in term 4, with + -- the votes of 0 and 3, and keeps the removal + periodicOne 2 100 + dispatchOne 2 + dispatchSingle 0 2 + dispatchSingle 3 2 + dispatchOne 2 + dispatchSingle 0 2 + dispatchSingle 3 2 + dispatchOne 2 + assertDetail 2 "leadership_state" "Leader" + assertLastTxid 2 "3.4" + + -- 2 appends a transaction at index 5, which replaces the signature on 4 + replicate 4 ["x"] + periodicOne 2 10 + dispatchSingle 2 4 + assertLastTxid 4 "4.5" + assertDetail 4 "membership_state" "Retired" + assertDetail 4 "retirement_phase" "Ordered" + assertCommitIdx 4 0 From c94fad0c10e8ceffc0eec7b379efb2846a1ed5a4 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 12:35:57 +0000 Subject: [PATCH 13/31] Shrink explored runs by the edges which they reach `--shrink PROGRAM OUTPUT --edges EDGES` leaves out steps of a program for as long as the model accepts its run, and the run reaches every edge which it reached at first and EDGES lack. An exploration writes the edges which the scenarios reach to `scenarios.edges`, for that use. Compile the model driver without inlining, so that the edges counted are those of cft.h only. With inlining, 8 of the 12 runs which a 20,000-run exploration kept for their edges added no line or branch of cft.h; without it, 1 of 4. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CMakeLists.txt | 5 ++- tests/cft_model/ModelDriver.lean | 8 +++-- tests/cft_model/ModelDriver/Explore.lean | 45 ++++++++++++++++++------ tests/cft_model/README.md | 10 +++--- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1988ea36e0e..d26626a1e26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1358,7 +1358,9 @@ if(BUILD_TESTS) add_warning_checks(cft_model_driver) enable_coverage(cft_model_driver) # Count the edges of the consensus implementation which a run reaches, - # so that the model driver can tell the runs which reach new ones + # so that the model driver can tell the runs which reach new ones. + # Without inlining, they are the edges of that implementation only, and + # not also those of the library code which it calls. if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(CFT_MODEL_EDGES ${CFT_MODEL_DIR}/edges.allowlist) target_compile_options( @@ -1366,6 +1368,7 @@ if(BUILD_TESTS) PRIVATE -fsanitize-coverage=inline-8bit-counters -fsanitize-coverage-allowlist=${CFT_MODEL_EDGES} + -fno-inline ) set_property( SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/aft/test/driver.cpp diff --git a/tests/cft_model/ModelDriver.lean b/tests/cft_model/ModelDriver.lean index 0ae8ef5f8d8..0b3ec528f38 100644 --- a/tests/cft_model/ModelDriver.lean +++ b/tests/cft_model/ModelDriver.lean @@ -242,9 +242,11 @@ def main (args : List String) : IO UInt32 := do | .error error => IO.eprintln error return 2 - | ["--shrink", program, output] => shrink program output none 8 + | ["--shrink", program, output] => shrink program output .finding 8 | ["--shrink", program, output, "--outcome", outcome] => - shrink program output (some outcome) 8 + shrink program output (.outcome outcome) 8 + | ["--shrink", program, output, "--edges", known] => + shrink program output (.edges (← IO.FS.readBinFile known)) 8 | [name] => if name == inStep then report "in step" runInStep @@ -254,5 +256,5 @@ def main (args : List String) : IO UInt32 := do | none => unknown name | _ => IO.eprintln - "usage: cft_model_driver SCENARIO | --in-step | --all | --record SCENARIO PROGRAM | --run PROGRAM [RESULT] | --explore DIRECTORY [--seed N] [--runs N] [--jobs N] | --shrink PROGRAM OUTPUT [--outcome OUTCOME] | --scenario PROGRAM NAME OUTPUT" + "usage: cft_model_driver SCENARIO | --in-step | --all | --record SCENARIO PROGRAM | --run PROGRAM [RESULT] | --explore DIRECTORY [--seed N] [--runs N] [--jobs N] | --shrink PROGRAM OUTPUT [--outcome OUTCOME | --edges EDGES] | --scenario PROGRAM NAME OUTPUT" return 2 diff --git a/tests/cft_model/ModelDriver/Explore.lean b/tests/cft_model/ModelDriver/Explore.lean index f8d35cae16a..cdb203c3db3 100644 --- a/tests/cft_model/ModelDriver/Explore.lean +++ b/tests/cft_model/ModelDriver/Explore.lean @@ -486,13 +486,23 @@ private def groups (jobs : Array Job) (size : Nat) : List (Array Job) := (List.range ((jobs.size + size - 1) / size)).map fun group => jobs.extract (group * size) ((group + 1) * size) +/-- What a shorter program must keep of the run of the program which it shortens. -/ +inductive Keep where + /-- The finding: its kind and its message, numbers aside. -/ + | finding + /-- An outcome which the model shows, in a run which the model accepts. -/ + | outcome (name : String) + /-- + Every edge which the run reaches and these lack, each nonzero if reached, in a run + which the model accepts. + -/ + | edges (known : ByteArray) + /-- -Make the program in `file` shorter: leave out steps for as long as its run shows `outcome` -and is accepted by the model or, if no outcome is given, is a finding of the same kind -and message, numbers aside. Write the program left to `output`. +Make the program in `file` shorter: leave out steps for as long as its run keeps what +`keep` asks. Write the program left to `output`. -/ -def shrink (file output : System.FilePath) (outcome : Option String) (jobs : Nat) - : IO UInt32 := do +def shrink (file output : System.FilePath) (keep : Keep) (jobs : Nat) : IO UInt32 := do let directory ← IO.FS.createTempDir IO.FS.createDirAll (directory / "run") let run (programs : Array (Array Step)) : IO (Array Ran) := do @@ -505,10 +515,20 @@ def shrink (file output : System.FilePath) (outcome : Option String) (jobs : Nat return ran let given ← readProgram file let some first := (← run #[given])[0]? | return 1 + let wanted := + match keep with + | .edges known => + (List.range first.edges.size).filter + fun index => first.edges.get! index != 0 && byte known index == 0 + | _ => [] let keeps (ran : Ran) : Bool := - match outcome with - | some outcome => ran.status == "ok" && ran.outcomes.contains outcome - | none => + match keep with + | .outcome name => ran.status == "ok" && ran.outcomes.contains name + | .edges _ => + ran.status == "ok" + && !wanted.isEmpty + && wanted.all fun index => byte ran.edges index != 0 + | .finding => -- The same finding, and still before or after the driver confirms progress ran.status != "ok" && ran.status != "invalid" @@ -547,9 +567,10 @@ def shrink (file output : System.FilePath) (outcome : Option String) (jobs : Nat size := (size + 1) / 2 IO.FS.removeDirAll directory let kept := - match outcome with - | some outcome => [s!"shows {outcome}"] - | none => [s!"{last.kind} at '{last.command}'", last.message.replace "\n" " "] + match keep with + | .outcome name => [s!"shows {name}"] + | .edges _ => [s!"reaches {wanted.length} edges which the others do not"] + | .finding => [s!"{last.kind} at '{last.command}'", last.message.replace "\n" " "] IO.FS.writeFile output (programText (s!"{file}, shrunk" :: kept) steps) IO.println s!"{given.size} steps shrunk to {steps.size}: {kept.head!}" return 0 @@ -592,6 +613,8 @@ def explore (options : Exploration) (scenarios : List String) : IO UInt32 := do edges := mergeEdges state.edges ran.edges } state := { state with shown := state.outcomes, reached := countEdges state.edges } + -- For `shrink`, to keep what a run reaches and the scenarios do not + IO.FS.writeBinFile (directory / "scenarios.edges") state.edges for jobs in groups seeds options.jobs do for job in jobs, ran in (← runJobs directory jobs) do state := (← absorb directory state job ran).1 diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index 56cc7c74d32..edf7bcfe331 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -58,15 +58,17 @@ It runs each changed program in a process of its own. Findings with the same message, numbers aside, are counted together. - A run is neither if the model does not cover it, for example because a primary is asked to remove a node which has not retired. -The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, and `report.md`. +The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, the edges which the scenarios reach in `scenarios.edges`, and `report.md`. The same seed gives the same changes. ```sh -build/cft_model_driver --shrink [--outcome ] +build/cft_model_driver --shrink [--outcome | --edges ] build/cft_model_driver --scenario .lean ``` -- `--shrink` leaves out steps of a program for as long as its run is the same finding or, with `--outcome`, still shows that outcome. +- `--shrink` leaves out steps of a program for as long as its run is the same finding. + With `--outcome`, it does so for as long as the model accepts the run and shows that outcome. + With `--edges`, it does so for as long as the model accepts the run, and the run reaches every edge which it reached at first and the edges in that file lack, such as `scenarios.edges`. - `--scenario` writes a program as a scenario in Lean, which ends by asserting the leadership and the commit index of each node. ## Building @@ -91,4 +93,4 @@ For the model driver, `lake` builds the consensus model, and fetches from GitHub - [`Command.lean`](ModelDriver/Command.lean): driver commands as data, programs, and scenarios written from them. - [`Episode.lean`](ModelDriver/Episode.lean): one run of the driver in a process, checked against the model. - [`Explore.lean`](ModelDriver/Explore.lean): the changes made to programs, the runs kept, and the findings. -- [`edges.allowlist`](edges.allowlist): the functions whose edges the driver counts, for which `driver.cpp` is compiled with `-fsanitize-coverage`. +- [`edges.allowlist`](edges.allowlist): the functions whose edges the driver counts, for which `driver.cpp` is compiled with `-fsanitize-coverage`, and without inlining, so that the edges are those of `cft.h` only. From 4ee7d65a6a39a01bb22b5b9ef4d84449ed63b581 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 14 Nov 2026 12:35:58 +0000 Subject: [PATCH 14/31] Add consensus scenarios for branches found by exploration Three runs which an exploration kept for the edges they reach each take a branch of cft.h which no scenario took: - stale_nack_after_ack: a rejection below the match index, in recv_append_entries_response - commit_index_beyond_batch: no committable index up to the one given, in commit_if_possible - nominate_over_retiring_node: a tie in match index broken by configuration, in find_successor The scenarios now miss 310 of the 859 branches of cft.h, from 313. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/cft_scenarios/Scenarios.lean | 3 ++ .../Scenarios/commit_index_beyond_batch.lean | 36 +++++++++++++++ .../nominate_over_retiring_node.lean | 46 +++++++++++++++++++ .../Scenarios/stale_nack_after_ack.lean | 46 +++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 tests/cft_scenarios/Scenarios/commit_index_beyond_batch.lean create mode 100644 tests/cft_scenarios/Scenarios/nominate_over_retiring_node.lean create mode 100644 tests/cft_scenarios/Scenarios/stale_nack_after_ack.lean diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index f57a3e7fd16..a1bd5d30edb 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -9,6 +9,7 @@ import Scenarios.check_quorum import Scenarios.check_quorum_012_0 import Scenarios.check_quorum_0_012 import Scenarios.commit_before_pending_reconfiguration +import Scenarios.commit_index_beyond_batch import Scenarios.commit_stepwise_through_reconfiguration import Scenarios.delayed_append_entries import Scenarios.delayed_vote_requests @@ -27,6 +28,7 @@ import Scenarios.matching_partial import Scenarios.message_type_summaries import Scenarios.multi_election import Scenarios.nominate_new_node +import Scenarios.nominate_over_retiring_node import Scenarios.nominate_successor import Scenarios.nominate_without_successor import Scenarios.pre_vote @@ -60,6 +62,7 @@ import Scenarios.rollback_safety import Scenarios.soft_rollback import Scenarios.sole_member_election import Scenarios.stale_ack_after_reelection +import Scenarios.stale_nack_after_ack import Scenarios.stale_vote_after_new_term import Scenarios.stale_vote_request import Scenarios.startup diff --git a/tests/cft_scenarios/Scenarios/commit_index_beyond_batch.lean b/tests/cft_scenarios/Scenarios/commit_index_beyond_batch.lean new file mode 100644 index 00000000000..db4e123f458 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/commit_index_beyond_batch.lean @@ -0,0 +1,36 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node which joins is told of a commit index beyond the entries which it has +-- received. The primary sends its log in two batches, and the first ends in a +-- transaction which follows a signature. Node 1 commits that signature as it +-- applies it, and has nothing further to commit at the end of the batch. +-- Found by `cft_model_driver --explore`, from `batched_append`, then written +-- to deliver one batch at a time. +initialize scenario "commit_index_beyond_batch" do + startNode 0 + replicate 2 [signature, "hello"] + replicate 2 [signature] + assertCommitIdx 0 4 + + -- 1 joins, holds nothing, and rejects the first message of the primary + trustNode 2 1 + dispatchAll + periodicOne 0 10 + + -- The first batch ends at 2.3, and the primary has committed 2.4 + dispatchSingle 0 1 + assertLastTxid 1 "2.3" + assertCommitIdx 1 2 + + -- The second batch ends at 2.5 + dispatchSingle 0 1 + assertLastTxid 1 "2.5" + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 4 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 4 diff --git a/tests/cft_scenarios/Scenarios/nominate_over_retiring_node.lean b/tests/cft_scenarios/Scenarios/nominate_over_retiring_node.lean new file mode 100644 index 00000000000..3528d389ba4 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/nominate_over_retiring_node.lean @@ -0,0 +1,46 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A primary which nominates a successor passes over a node which is being +-- removed. Nodes 1, 2 and 3 have acknowledged the same entries when the +-- primary orders the removal of node 3, so node 3 is the only one which is not +-- in the latest configuration. The primary proposes node 2, which is elected. +-- Found by `cft_model_driver --explore`, from `retire_backup_pre_vote`, then +-- written to elect the nominee. +initialize scenario "nominate_over_retiring_node" do + startNode 0 + trustNodes 2 [1, 2, 3] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + -- Initial network state [0, 1, 2, 3], committed at 2.3 + + -- The primary orders the removal of 3 at 2.4, which no other node receives + swapNodes 2 [3] [] + assertDetail 0 "leadership_state" "Leader" + + -- The primary nominates 2, which campaigns in term 3 + nominateSuccessor 0 + dispatchSingle 0 2 + assertDetail 2 "leadership_state" "Candidate" + assertDetail 3 "leadership_state" "Follower" + + -- 2 is elected with the votes of 0 and 1 + dispatchOne 2 + dispatchSingle 0 2 + dispatchSingle 1 2 + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 3 + assertDetail 2 "leadership_state" "Leader" + assertCommitIdx 2 3 + assertDetail 3 "leadership_state" "Follower" + assertCommitIdx 3 3 diff --git a/tests/cft_scenarios/Scenarios/stale_nack_after_ack.lean b/tests/cft_scenarios/Scenarios/stale_nack_after_ack.lean new file mode 100644 index 00000000000..dccbe001081 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/stale_nack_after_ack.lean @@ -0,0 +1,46 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A rejection which arrives after the acknowledgement of later entries does +-- not send the primary back. Node 2 rejects the first message which the +-- primary sends to it, and that rejection is delayed until node 2 has +-- acknowledged the whole log. The primary then sends node 2 only the entries +-- which follow. +-- Found by `cft_model_driver --explore`, from `reconfiguration`, then written +-- to end with a commit. +initialize scenario "stale_nack_after_ack" do + startNode 0 + trustNodes 2 [1, 2] + + -- 1 and 2 hold nothing, and reject the first message of the primary. The + -- rejection from 2 is held back + dispatchOne 0 + holdNth 2 0 0 + + -- 1 and 2 receive the log up to 2.2, and acknowledge it + periodicOne 0 10 + dispatchAll + periodicOne 0 10 + dispatchAll + assertLastTxid 2 "2.2" + + -- The rejection from 2 arrives, and names an index below the one which 2 has + -- acknowledged + dispatchHeld 0 + + -- The primary signs at 2.3, which commits on every node + replicate 2 [signature] + periodicOne 0 10 + dispatchAll + periodicOne 0 10 + dispatchAll + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 3 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 3 + assertDetail 2 "leadership_state" "Follower" + assertCommitIdx 2 3 From 6b7ff8449c65a2b73928031ae8147607474fe156 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 08:03:42 +0000 Subject: [PATCH 15/31] Refuse a consensus configuration with no nodes on the primary A transaction which retires the last trusted node of a service leaves a configuration with no nodes. No quorum of it exists, so nothing can be committed once it is active: update_commit throws std::out_of_range on it. A sole primary threw from replicate after it had appended the entry and passed it to its ledger. A primary with other nodes sent them the entry: in a driver run with three nodes, every node adopted it and none committed anything after it. add_configuration now throws on a primary which is given an empty configuration, before it changes anything, so replicate fails before the entry is appended or sent. The node still stops, because the KV treats a failure to replicate a transaction which it has applied as fatal. Rolling the transaction back instead would stay in the same view, and let its sequence number be used again. Other nodes still adopt what their primary sends. A single-node service stopped with 'Failed to serialise committed transaction' when its node was removed, before this change and after it. Before it, the ledger file of the node held the first 4096 bytes of the entry which removes it. After it, the node logs why it refuses, and writes no part of that entry. The model already disables this append, so an explored run which asks for it is now set aside, where it was a finding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .changes/refuse-empty-configuration.fixed.md | 2 + doc/governance/common_member_operations.rst | 4 ++ src/consensus/aft/cft.h | 15 +++++++ src/consensus/aft/test/main.cpp | 47 ++++++++++++++++++++ tests/cft_model/ModelDriver/Episode.lean | 12 +++-- tests/cft_model/README.md | 2 +- tests/nodes.py | 1 + tests/reconfiguration.py | 42 +++++++++++++++++ 8 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 .changes/refuse-empty-configuration.fixed.md diff --git a/.changes/refuse-empty-configuration.fixed.md b/.changes/refuse-empty-configuration.fixed.md new file mode 100644 index 00000000000..f894fc941b2 --- /dev/null +++ b/.changes/refuse-empty-configuration.fixed.md @@ -0,0 +1,2 @@ +A primary now refuses a transaction which would leave the service with no trusted node before it writes that transaction to its ledger or sends it to another node, where it previously recorded a configuration after which nothing could be committed. +The node still stops with a fatal error; see [Removing an Existing Node](doc/governance/common_member_operations.rst#removing-an-existing-node). diff --git a/doc/governance/common_member_operations.rst b/doc/governance/common_member_operations.rst index 1ffa3975c91..8e2c2e98330 100644 --- a/doc/governance/common_member_operations.rst +++ b/doc/governance/common_member_operations.rst @@ -21,6 +21,10 @@ The operator can establish if it is safe to remove a node by calling :http:GET:` .. note:: If the now-retired node was the primary node, once the proposal successfully completes, an election will take place to elect a new primary. +.. warning:: + Do not remove the last trusted node of a service. + Nothing could be committed once no trusted node is left, so the primary refuses the transaction which removes it, and stops with a fatal error before it writes that transaction to its ledger. + Updating Code Version --------------------- diff --git a/src/consensus/aft/cft.h b/src/consensus/aft/cft.h index 555b4059102..44bed14ae39 100644 --- a/src/consensus/aft/cft.h +++ b/src/consensus/aft/cft.h @@ -557,6 +557,21 @@ namespace aft CFT_DEBUG_FMT( "Configurations: add new configuration at {}: {{{}}}", idx, conf); + // A configuration with no nodes has no quorum, so nothing could be + // committed once it is active. A primary refuses the transaction which + // asks for one, before it appends the entry or sends it to another + // node. Other nodes must follow the log which their primary sends. + if ( + conf.empty() && + state->leadership_state.load() == ccf::kv::LeadershipState::Leader) + { + CFT_FAIL_FMT( + "Refusing to replicate the configuration at {}: it has no nodes", + idx); + throw std::logic_error( + fmt::format("Configuration at {} has no nodes", idx)); + } + trace::add_configuration(*state, configurations, idx, conf, idx); // Detect when we are retired by observing a configuration diff --git a/src/consensus/aft/test/main.cpp b/src/consensus/aft/test/main.cpp index e49147a0b39..84bdbc3ae96 100644 --- a/src/consensus/aft/test/main.cpp +++ b/src/consensus/aft/test/main.cpp @@ -3170,6 +3170,53 @@ DOCTEST_TEST_CASE( DOCTEST_REQUIRE(r0.ledger->ledger.size() == 1); } +DOCTEST_TEST_CASE( + "A primary refuses a configuration with no nodes" * + doctest::test_suite("single")) +{ + ccf::NodeId node_id = ccf::kv::test::PrimaryNodeId; + TestNode node(node_id); + auto& r0 = node.cft; + + aft::Configuration::Nodes config; + config[node_id] = {}; + r0.add_configuration(0, config); + + r0.start_ticking(); + r0.periodic(election_timeout * 2); + DOCTEST_REQUIRE(r0.is_primary()); + + auto entry = + std::make_shared>(std::vector{1, 2, 3}); + DOCTEST_REQUIRE( + r0.replicate(ccf::kv::BatchVector{{1, entry, true, hooks}}, 1)); + DOCTEST_REQUIRE(r0.get_committed_seqno() == 1); + + DOCTEST_INFO("The transaction which removes the only node is refused"); + auto removal_hooks = std::make_shared(); + removal_hooks->push_back(std::make_unique( + aft::Configuration::Nodes{}, 2)); + DOCTEST_REQUIRE_THROWS_WITH_AS( + r0.replicate(ccf::kv::BatchVector{{2, entry, false, removal_hooks}}, 1), + "Configuration at 2 has no nodes", + std::logic_error); + + DOCTEST_INFO("Nothing is appended, and the configuration is as it was"); + DOCTEST_REQUIRE(r0.get_last_idx() == 1); + DOCTEST_REQUIRE(r0.ledger->ledger.size() == 1); + const auto details = r0.get_details(); + DOCTEST_REQUIRE(details.configs.size() == 1); + DOCTEST_REQUIRE(details.configs.back().nodes == config); + DOCTEST_REQUIRE(details.membership_state == ccf::kv::MembershipState::Active); + DOCTEST_REQUIRE(r0.is_primary()); + + DOCTEST_INFO("A backup follows the log of its primary, as before"); + TestNode backup(ccf::kv::test::FirstBackupNodeId); + backup.cft.add_configuration(0, config); + backup.cft.add_configuration(2, {}); + DOCTEST_REQUIRE(backup.cft.get_details().configs.size() == 2); +} + int main(int argc, char** argv) { doctest::Context context; diff --git a/tests/cft_model/ModelDriver/Episode.lean b/tests/cft_model/ModelDriver/Episode.lean index 515c112a609..e470a5bd312 100644 --- a/tests/cft_model/ModelDriver/Episode.lean +++ b/tests/cft_model/ModelDriver/Episode.lean @@ -69,7 +69,7 @@ def checkPending (session : Session) : IO Unit := do /-- Why the driver stopped before the end of a run. -/ structure Stop where - /-- `stuck`, `invariant` or `error`. -/ + /-- `stuck`, `invariant`, `refused` or `error`. -/ kind : String message : String /-- The command which it could not run. -/ @@ -80,7 +80,8 @@ structure Report where /-- `ok`, or `finding` if the driver and the model disagree or the driver fails, or `invalid` if the model does not cover the run: it does not start a network, or it has a primary - append what no application would, such as the removal of a node which has not retired. + append what no application would, such as the removal of a node which has not retired, + or of every node, which CCF refuses. -/ status : String := "ok" /-- What kind of finding: `model`, `undelivered`, `invariant` or `error`. -/ @@ -124,6 +125,9 @@ private def stopFor (error : String) : Option String := none else if has error ["at unsafe commit idx"] then some "invariant" + else if has error ["has no nodes"] then + -- A primary refuses a configuration with no nodes, which the model disables + some "refused" else if isStuck error then some "stuck" else @@ -177,7 +181,7 @@ private def conclude (stopped : Option Stop) let counters ← edges -- A command which fails may leave a group of events unfinished. let whole := - if stopped.any (·.kind == "error") then + if stopped.any (fun stop => stop.kind == "error" || stop.kind == "refused") then (splitCommands records).pop.foldl (· ++ ·) #[] else records @@ -210,6 +214,8 @@ private def conclude (stopped : Option Stop) let report : Report := if stop.kind == "stuck" then { outcomes, checks, stuck := stop.message } + else if stop.kind == "refused" then + { status := "invalid", message := stop.message, command := stop.command } else { status := "finding" diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index edf7bcfe331..8d6c4d7a164 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -56,7 +56,7 @@ It runs each changed program in a process of its own. It is run a second time first, and is only kept if it does so again. - A run is a finding if the driver fails, for example because a commit is unsafe, or if the model rejects its trace. Findings with the same message, numbers aside, are counted together. -- A run is neither if the model does not cover it, for example because a primary is asked to remove a node which has not retired. +- A run is neither if the model does not cover it, for example because a primary is asked to remove a node which has not retired, or to remove every node, which CCF refuses. The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, the edges which the scenarios reach in `scenarios.edges`, and `report.md`. The same seed gives the same changes. diff --git a/tests/nodes.py b/tests/nodes.py index 1be3b3f7365..7c456f3c6ff 100644 --- a/tests/nodes.py +++ b/tests/nodes.py @@ -339,6 +339,7 @@ def add(parser): "join_no_snapshot_ipv6", reconfiguration.run_join_no_snapshot_against_original_primary_ipv6, ), + ("remove_last_node", reconfiguration.run_remove_last_node), ): cr.add( name, diff --git a/tests/reconfiguration.py b/tests/reconfiguration.py index 1ece407f079..d2c4c976d6c 100644 --- a/tests/reconfiguration.py +++ b/tests/reconfiguration.py @@ -1465,6 +1465,48 @@ def run_join_no_snapshot_against_original_primary(const_args, ipv6=False): ), f"Joiner should have started from a fetched snapshot, got startup_seqno={body['startup_seqno']}" +def run_remove_last_node(const_args): + # A service which removed its last trusted node could commit nothing + # afterwards. The primary refuses the transaction which does so before it + # writes it to its ledger, and stops, as it does for any transaction which + # it has applied and cannot replicate. + args = deepcopy(const_args) + args.nodes = infra.e2e_args.nodes(args, 1) + args.label += "_remove_last_node" + + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + pdb=args.pdb, + ) as network: + network.start_and_open(args) + primary, backups = network.find_nodes() + assert not backups, backups + + # The node reports a fatal error as it stops + network.ignore_errors_on_shutdown() + try: + network.consortium.retire_node(primary, primary) + except ( + infra.clients.CCFConnectionException, + infra.clients.CCFIOException, + TimeoutError, + ): + LOG.info("The node did not answer the vote which removes it") + else: + raise AssertionError("The last trusted node of the service was removed") + + assert primary.remote.check_done(timeout=10), "The node is still running" + + out_path, _ = primary.get_logs() + refusal = "Refusing to replicate the configuration" + with open(out_path, encoding="utf-8") as out: + assert any( + refusal in line for line in out + ), f"Expected '{refusal}' in {out_path}" + + def _assert_ipv6_available(): assert infra.net.ipv6_loopback_available(), ( "IPv6 loopback (::1) is not available; CI enables IPv6 via the " From 12908658bc85f1a182bd64f94a3c155c7ae44125 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 08:04:14 +0000 Subject: [PATCH 16/31] Order retirement confirmations in the consensus model as CCF does When a commit confirms the retirement of a primary, CCF nominates a successor. set_retired_committed takes the nodes which a transaction confirms in the order of their identifiers, and forgets each node other than itself as it does. So the primary no longer knows the nodes which were confirmed before it: by an earlier transaction of the commit, or by the same transaction with a lower identifier. It still knows, and may nominate, a node which the same transaction confirms with a higher identifier. The model nominated among every peer which the primary knew before the commit. It disagreed with CCF when a node confirmed before the primary had acknowledged more than the nodes which stay. advanceCommit now nominates with nominateOnRetirement, which leaves out retirementsConfirmedBefore. cft.h is unchanged. findSuccessor is now bestSuccessor over the eligible peers, and means what it did. The proof that it equals its definition with Mathlib unfolds one more name. There is a scenario for each order of the identifiers. The model rejected the trace of retirement_forgets_lower_node before this change. It accepts that of retirement_nominates_higher_node before and after it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Consensus/Model/ClusterConfig.lean | 11 +++ lean/consensus/Consensus/Model/Node.lean | 26 ++++- lean/consensus/Consensus/Proofs/Deadlock.lean | 10 +- .../Consensus/Proofs/SafetyInvariant.lean | 20 +++- .../MathlibEquivalence/Rank.lean | 2 +- tests/cft_scenarios/Scenarios.lean | 2 + .../retirement_forgets_lower_node.lean | 79 +++++++++++++++ .../retirement_nominates_higher_node.lean | 96 +++++++++++++++++++ 8 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/retirement_forgets_lower_node.lean create mode 100644 tests/cft_scenarios/Scenarios/retirement_nominates_higher_node.lean diff --git a/lean/consensus/Consensus/Model/ClusterConfig.lean b/lean/consensus/Consensus/Model/ClusterConfig.lean index 116a8cc54d9..d4bef908109 100644 --- a/lean/consensus/Consensus/Model/ClusterConfig.lean +++ b/lean/consensus/Consensus/Model/ClusterConfig.lean @@ -83,6 +83,17 @@ def retiredCommittedNodes (log : List Entry) : Finset NodeID := log.foldl (fun retired entry => retired ∪ entry.content.metadata.retirementConfirmations) ∅ +/-- +Nodes whose retirement the log confirms before that of `node`: in an earlier +transaction, or in the same transaction with a smaller identifier. CCF confirms the +retirements which a commit covers in that order. +-/ +def retirementsConfirmedBefore (log : List Entry) (node : NodeID) : Finset NodeID := + let confirmations := log.map (·.content.metadata.retirementConfirmations) + let earlier := confirmations.takeWhile fun nodes => decide (node ∉ nodes) + let same := (confirmations.drop earlier.length).headD ∅ + earlier.foldl (· ∪ ·) ∅ ∪ same.filter (· < node) + def hasConfigurationMajority (support : Finset NodeID) (configuration : Configuration) : Prop := (support ∩ configuration.nodes).card * 2 > configuration.nodes.card diff --git a/lean/consensus/Consensus/Model/Node.lean b/lean/consensus/Consensus/Model/Node.lean index 413689db3d2..270ff7f591f 100644 --- a/lean/consensus/Consensus/Model/Node.lean +++ b/lean/consensus/Consensus/Model/Node.lean @@ -64,12 +64,11 @@ def rankBelow (a b : Nat × Nat × Nat) : Bool := decide (a.1 < b.1 \/ (a.1 = b.1 /\ Prod.lexLt a.2 b.2)) /-- Maximize replication progress, latest active membership, then node ID. -/ -def findSuccessor (state : NodeState) (self : NodeID) : Option NodeID := +def bestSuccessor (state : NodeState) (peers : Finset NodeID) : Option NodeID := let configurations := state.activeConfigurations let rank node := (state.matchIndex node, latestMembershipIndex configurations node, node) - let peers := (eligiblePeers state self).sort - peers.foldl + peers.sort.foldl (fun best candidate => match best with | none => some candidate @@ -80,6 +79,9 @@ def findSuccessor (state : NodeState) (self : NodeID) : Option NodeID := best) none +def findSuccessor (state : NodeState) (self : NodeID) : Option NodeID := + bestSuccessor state (eligiblePeers state self) + def nominateSuccessor (caps : Caps) (state : NodeState) (self : NodeID) : NodeEffect NodeState := do guard (state.role = .primary) @@ -87,6 +89,19 @@ def nominateSuccessor (caps : Caps) (state : NodeState) (self : NodeID) caps.send (.proposeRequestVoteRequest state.currentTerm) successor return state +/-- +Nominate as the retirement of this primary is confirmed. CCF has by then forgotten the +nodes whose retirement it confirmed first, and nominates none of them. +-/ +def nominateOnRetirement (caps : Caps) (state : NodeState) (self : NodeID) + : NodeEffect NodeState := do + guard (state.role = .primary) + if let some successor := + bestSuccessor state + (eligiblePeers state self \ retirementsConfirmedBefore state.log self) then + caps.send (.proposeRequestVoteRequest state.currentTerm) successor + return state + abbrev canStartElection (state : NodeState) (self : NodeID) : Prop := (state.role = .follower \/ state.role = .preVoteCandidate \/ state.role = .candidate) /\ (self ∈ configurationMembers state.signedActiveConfigurations @@ -140,7 +155,8 @@ def NodeState.recordCommit (node : NodeID) (state : NodeState) (commitIndex : Na /-- Retirement nominates using the precommit peer and configuration view, before -discarding old configurations and clearing the outgoing primary's role. +discarding old configurations and clearing the outgoing primary's role. Peers +whose retirement CCF confirms first are forgotten by then. -/ def advanceCommit (caps : Caps) (state : NodeState) (self : NodeID) : NodeEffect NodeState := do @@ -152,7 +168,7 @@ def advanceCommit (caps : Caps) (state : NodeState) (self : NodeID) let state := state.recordCommit self commitIndex if state.isRetiredCommitted self then if preCommitState.role = .primary then - let _ ← nominateSuccessor caps preCommitState self + let _ ← nominateOnRetirement caps preCommitState self return state else return state diff --git a/lean/consensus/Consensus/Proofs/Deadlock.lean b/lean/consensus/Consensus/Proofs/Deadlock.lean index a7115642ab2..787356a534f 100644 --- a/lean/consensus/Consensus/Proofs/Deadlock.lean +++ b/lean/consensus/Consensus/Proofs/Deadlock.lean @@ -80,13 +80,21 @@ private theorem nominateSuccessor (state : NodeState) (primary : state.role = .p · exact bind' (send _ _ _) fun _ => pure' _ · exact pure' _ +private theorem nominateOnRetirement (state : NodeState) (primary : state.role = .primary) + : Total (Model.Node.nominateOnRetirement (Capabilities.record self) state self) := by + unfold Model.Node.nominateOnRetirement + refine guard_bind primary ?_ + split + · exact bind' (send _ _ _) fun _ => pure' _ + · exact pure' _ + private theorem advanceCommit (state : NodeState) : Total (Model.Node.advanceCommit (Capabilities.record self) state self) := by unfold Model.Node.advanceCommit refine ite' (fun _ => pure' _) fun _ => ?_ refine ite' (fun _ => ?_) fun _ => pure' _ refine ite' (fun primary => ?_) fun _ => pure' _ - exact bind' (nominateSuccessor self state primary) fun _ => pure' _ + exact bind' (nominateOnRetirement self state primary) fun _ => pure' _ private theorem broadcastToOthers (state : NodeState) (message : Message) : Total diff --git a/lean/consensus/Consensus/Proofs/SafetyInvariant.lean b/lean/consensus/Consensus/Proofs/SafetyInvariant.lean index e475a98aa09..1c09a944822 100644 --- a/lean/consensus/Consensus/Proofs/SafetyInvariant.lean +++ b/lean/consensus/Consensus/Proofs/SafetyInvariant.lean @@ -3586,6 +3586,24 @@ private theorem nominate_sends (state : NodeState) Sends.pure' ⟨rfl, leader, by simp [eq]⟩ · exact Sends.pure' ⟨rfl, leader, by simp⟩ +private theorem nominateOnRetirement_sends (state : NodeState) + : Sends + (fun result sent => + result = state + /\ state.role = .primary + /\ forall e, + e ∈ sent + -> e.source = self + /\ e.payload = .proposeRequestVoteRequest state.currentTerm) + (nominateOnRetirement (Capabilities.record self : Caps) state self) := by + unfold nominateOnRetirement + refine Sends.bind' Sends.guard' fun _ _ ⟨leader, empty⟩ => ?_ + subst empty + split + · exact Sends.bind' (Sends.send _ _ _) fun _ sent eq => + Sends.pure' ⟨rfl, leader, by simp [eq]⟩ + · exact Sends.pure' ⟨rfl, leader, by simp⟩ + private theorem advanceCommit_sends (middle : NodeState) : Sends (fun state sent => @@ -3608,7 +3626,7 @@ private theorem advanceCommit_sends (middle : NodeState) refine Sends.ite' (fun _ => Sends.ite' (fun _ => ?_) fun _ => Sends.pure' ⟨advanced, by simp⟩) fun _ => Sends.pure' ⟨advanced, by simp⟩ - exact Sends.bind' (nominate_sends self middle) + exact Sends.bind' (nominateOnRetirement_sends self middle) fun _ sent nominated => Sends.pure' ⟨advanced, by simpa using nominated.2.2⟩ diff --git a/lean/mathlib-equivalence/MathlibEquivalence/Rank.lean b/lean/mathlib-equivalence/MathlibEquivalence/Rank.lean index e5ae27b72e2..dd003f95f47 100644 --- a/lean/mathlib-equivalence/MathlibEquivalence/Rank.lean +++ b/lean/mathlib-equivalence/MathlibEquivalence/Rank.lean @@ -39,7 +39,7 @@ def findSuccessorLex (state : NodeState) (self : NodeID) : Option NodeID := theorem findSuccessor_eq_lex (state : NodeState) (self : NodeID) : findSuccessor state self = findSuccessorLex state self := by - simp only [findSuccessor, findSuccessorLex, rankBelow_eq, decide_eq_true_eq] + simp only [findSuccessor, bestSuccessor, findSuccessorLex, rankBelow_eq, decide_eq_true_eq] congr! end MathlibEquivalence.Rank diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index a1bd5d30edb..c43010acce6 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -55,6 +55,8 @@ import Scenarios.retire_one import Scenarios.retired_committed_delayed_nomination import Scenarios.retired_node_delayed_responses import Scenarios.retirement_commit_below_signature +import Scenarios.retirement_forgets_lower_node +import Scenarios.retirement_nominates_higher_node import Scenarios.retirement_nominates_most_replicated import Scenarios.retirement_rollback import Scenarios.retirement_signature_rollback diff --git a/tests/cft_scenarios/Scenarios/retirement_forgets_lower_node.lean b/tests/cft_scenarios/Scenarios/retirement_forgets_lower_node.lean new file mode 100644 index 00000000000..9158e512a0f --- /dev/null +++ b/tests/cft_scenarios/Scenarios/retirement_forgets_lower_node.lean @@ -0,0 +1,79 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A primary which retires with a node of a lower identifier does not nominate +-- that node. One transaction confirms the retirement of nodes 0 and 2. CCF +-- confirms them in the order of their identifiers, and forgets each node +-- other than itself as it does. So node 2, the primary, has forgotten node 0 +-- when its own retirement is confirmed, and nominates node 1, although node 0 +-- has acknowledged more. +-- Found by `cft_model_driver --explore`, from +-- `retired_committed_delayed_nomination`, then written so that every node +-- learns of its retirement. +initialize scenario "retirement_forgets_lower_node" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + -- Initial network state [0, 1, 2], committed at 2.3 + + -- 2 is elected in term 3 + periodicOne 2 100 + dispatchOne 2 + dispatchOne 0 + dispatchOne 1 + dispatchOne 2 + dispatchOne 0 + dispatchOne 1 + dispatchOne 2 + dispatchOne 0 + dispatchOne 1 + assertDetail 2 "leadership_state" "Leader" + + -- 2 removes 0 and itself at 3.4, and signs at 3.5, which commits + swapNodes 3 [0, 2] [] + replicate 3 [signature] + periodicOne 2 10 + dispatchOne 2 + dispatchOne 0 + dispatchOne 1 + periodicOne 2 10 + dispatchOne 2 + dispatchOne 0 + dispatchOne 1 + assertCommitIdx 0 5 + assertCommitIdx 1 5 + assertDetail 0 "retirement_phase" "Completed" + assertDetail 2 "retirement_phase" "Completed" + + -- 2 confirms both retirements at 3.6, and signs at 3.7 and at 3.8 + cleanupNodes 3 [0, 2] + replicate 3 [signature] + periodicOne 2 10 + replicate 3 [signature] + periodicOne 2 10 + + -- 0 acknowledges 3.8, and 1 only 3.7, which commits the confirmation + dispatchSingle 2 0 + dispatchSingle 2 0 + dispatchOne 0 + dispatchSingle 2 1 + dispatchSingle 1 2 + assertCommitIdx 2 7 + assertDetail 2 "retirement_phase" "RetiredCommitted" + assertDetail 2 "leadership_state" "None" + + -- 2 has nominated 1, the only node left, which is elected in term 4 + dispatchOne 2 + assertDetail 1 "leadership_state" "Leader" + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 5 + assertCommitIdx 1 5 diff --git a/tests/cft_scenarios/Scenarios/retirement_nominates_higher_node.lean b/tests/cft_scenarios/Scenarios/retirement_nominates_higher_node.lean new file mode 100644 index 00000000000..bcf34a08285 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/retirement_nominates_higher_node.lean @@ -0,0 +1,96 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A primary which retires with a node of a higher identifier may nominate +-- that node. One transaction confirms the retirement of nodes 1 and 2. CCF +-- confirms them in the order of their identifiers, so node 1, the primary, +-- still knows node 2 when its own retirement is confirmed, and nominates it, +-- because it has acknowledged more than node 0. Node 2 is elected, commits +-- the confirmation in its turn, and nominates node 0, which stays. +-- Found by `cft_model_driver --explore`, from +-- `retired_committed_delayed_nomination`, then written to end with the node +-- which stays as primary. +initialize scenario "retirement_nominates_higher_node" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + -- Initial network state [0, 1, 2], committed at 2.3 + + -- 1 is elected in term 3 + periodicOne 1 100 + dispatchOne 1 + dispatchOne 0 + dispatchOne 2 + dispatchOne 1 + dispatchOne 0 + dispatchOne 2 + dispatchOne 1 + dispatchOne 0 + dispatchOne 2 + assertDetail 1 "leadership_state" "Leader" + + -- 1 removes itself and 2 at 3.4, and signs at 3.5, which commits + swapNodes 3 [1, 2] [] + replicate 3 [signature] + periodicOne 1 10 + dispatchOne 1 + dispatchOne 0 + dispatchOne 2 + periodicOne 1 10 + dispatchOne 1 + dispatchOne 0 + dispatchOne 2 + assertCommitIdx 0 5 + assertCommitIdx 2 5 + assertDetail 1 "retirement_phase" "Completed" + assertDetail 2 "retirement_phase" "Completed" + + -- 1 confirms both retirements at 3.6, and signs at 3.7 and at 3.8 + cleanupNodes 3 [1, 2] + replicate 3 [signature] + periodicOne 1 10 + replicate 3 [signature] + periodicOne 1 10 + + -- 2 acknowledges 3.8, and 0 only 3.7, which commits the confirmation + dispatchSingle 1 2 + dispatchSingle 1 2 + dispatchOne 2 + dispatchSingle 1 0 + dispatchSingle 0 1 + assertCommitIdx 1 7 + assertDetail 1 "retirement_phase" "RetiredCommitted" + assertDetail 1 "leadership_state" "None" + + -- 1 has nominated 2, which is elected in term 4 with the vote of 0 + dispatchSingle 1 0 + dispatchSingle 1 2 + dispatchOne 2 + dispatchOne 0 + assertDetail 2 "leadership_state" "Leader" + assertDetail 2 "retirement_phase" "Completed" + + -- 2 signs at 4.9, which commits the confirmation of its own retirement + replicate 4 [signature] + periodicOne 2 10 + dispatchOne 2 + dispatchOne 0 + assertCommitIdx 2 9 + assertDetail 2 "retirement_phase" "RetiredCommitted" + assertDetail 2 "leadership_state" "None" + + -- 2 has forgotten 1, and nominated 0, which is elected in term 5 + dispatchOne 2 + assertDetail 0 "leadership_state" "Leader" + + assertCommitIdx 0 5 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 7 From f77674c5a2e6ac48c4bdce3a9f8e96266d8d5890 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 08:04:31 +0000 Subject: [PATCH 17/31] Acknowledge AppendEntries ending before entries of a later term After a node has handled an AppendEntries which does not end in a signature, it records that the term of the last index starts after its last signature. If the node still held entries of a later term beyond that index, the view history refused the record with 'version must not move backwards'. recv_message logged the exception, and the node sent no response until an AppendEntries replaced those entries. The node now records nothing when it already knows the term of the last index, and acknowledges, as the model does. It keeps the later entries until the sender replaces them. This needs a primary which kept entries after its last signature when it was elected. become_leader only allows that while nothing is committed, so a running service should not get there. It was found by exploring changes to the scenarios, and heartbeat_before_stale_suffix fails without this change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/cft.h | 5 +- tests/cft_scenarios/Scenarios.lean | 1 + .../heartbeat_before_stale_suffix.lean | 76 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/cft_scenarios/Scenarios/heartbeat_before_stale_suffix.lean diff --git a/src/consensus/aft/cft.h b/src/consensus/aft/cft.h index 44bed14ae39..b87059e0fdf 100644 --- a/src/consensus/aft/cft.h +++ b/src/consensus/aft/cft.h @@ -1484,7 +1484,10 @@ namespace aft // The end of this append entries (r.idx) was not a signature, but may // be in a new term. If it's a new term, this term started immediately // after the previous signature we saw (lci, last committable index). - if (r.idx > lci) + // If we already know the term of r.idx, there is nothing to record. + // We may then still hold entries of a later term after r.idx, which + // the sender has yet to replace, and that term starts after lci + 1. + if (r.idx > lci && get_term_internal(r.idx) != r.term_of_idx) { state->view_history.update(lci + 1, r.term_of_idx); } diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index c43010acce6..8437877c852 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -19,6 +19,7 @@ import Scenarios.fancy_election_1 import Scenarios.fancy_election_2 import Scenarios.follower_overcommit_partial_verified import Scenarios.follower_rollback_match_index +import Scenarios.heartbeat_before_stale_suffix import Scenarios.indirect_commit_stale_leader import Scenarios.large_entry_batching import Scenarios.late_nomination diff --git a/tests/cft_scenarios/Scenarios/heartbeat_before_stale_suffix.lean b/tests/cft_scenarios/Scenarios/heartbeat_before_stale_suffix.lean new file mode 100644 index 00000000000..708450cb533 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/heartbeat_before_stale_suffix.lean @@ -0,0 +1,76 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- A follower acknowledges a heartbeat which ends before an entry of a later +-- term that it still holds. Nothing is committed, so a node which is elected +-- keeps the transaction after its last signature. Node 1 appends an entry in +-- term 3 after that transaction, and node 0 is then elected in term 4 without +-- it. The first heartbeat of node 0 ends at the transaction, whose term node 1 +-- knows already, and before the entry of term 3. +-- Found by `cft_model_driver --explore`, from `retirement_signature_rollback`, +-- then written for two nodes. +initialize scenario "heartbeat_before_stale_suffix" do + startNode 0 + trustNode 2 1 + replicate 2 [signature, "x"] + dispatchAll + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "2.4" + -- Initial network state [0, 1]. The acknowledgements of 1 are lost, so + -- nothing is committed + dropPending 1 + + -- 1 is elected in term 3, and keeps 2.4, which is not signed + periodicOne 1 100 + dispatchOne 1 + dispatchOne 0 + dispatchOne 1 + dispatchOne 0 + assertDetail 1 "leadership_state" "Leader" + assertLastTxid 1 "2.4" + dropPending 1 + + -- 1 appends 3.5, which 0 does not receive + replicate 3 ["y"] + assertLastTxid 1 "3.5" + + -- 0 is elected in term 4, and keeps 2.4 too + periodicOne 0 100 + dispatchOne 0 + dispatchOne 1 + dispatchOne 0 + dispatchOne 1 + assertDetail 0 "leadership_state" "Leader" + assertDetail 1 "leadership_state" "Follower" + assertLastTxid 0 "2.4" + assertPending 1 0 0 + + -- The first heartbeat of 0 ends at 2.4. 1 acknowledges it, and keeps 3.5 + dispatchOne 0 + assertLastTxid 1 "3.5" + assertPending 1 0 1 + dispatchOne 1 + + -- 0 signs at 4.5, which replaces 3.5 on 1, and commits + replicate 4 [signature] + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "4.5" + dispatchOne 1 + periodicOne 0 10 + dispatchOne 0 + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 5 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 5 From 0707298e6ad2c3d66d9ee245aeec73600643a56d Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 08:25:47 +0000 Subject: [PATCH 18/31] Do not require a scenario and its program to show the same outcomes cft_model_test runs each scenario again from its recorded commands, and required both runs to show the same model outcomes. The election timeout of a node differs between processes, so they need not: retirement_nominates_most_replicated leaves node 3 without a primary for 60 ms, and node 3 started an election in about 1 pair of runs in 40, which failed the test. The program must now be accepted by the model and issue every recorded command, the assertions of the scenario among them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/cft_model/ModelDriver.lean | 12 +++++++----- tests/cft_model/README.md | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/cft_model/ModelDriver.lean b/tests/cft_model/ModelDriver.lean index 0b3ec528f38..9dc351abe93 100644 --- a/tests/cft_model/ModelDriver.lean +++ b/tests/cft_model/ModelDriver.lean @@ -124,18 +124,20 @@ private def again (args : Array String) : IO (UInt32 × String) := do let errors ← child.stderr.readToEnd return (← child.wait, errors) -/-- The status and the outcomes in the result file of a run, and its message. -/ +/-- The status and the commands in the result file of a run, and its message. -/ private def reported (file : System.FilePath) : IO (String × Json × String) := do let result ← IO.ofExcept (Json.parse (← IO.FS.readFile file)) return ( ← IO.ofExcept (string result "status"), - ← IO.ofExcept (field result "outcomes"), + ← IO.ofExcept (field result "commands"), ← IO.ofExcept (string result "message") ) /-- -Record the commands of a scenario, and run them as a program, which must show the outcomes -which the scenario shows. Give what went wrong, if anything did. +Record the commands of a scenario, and run them as a program, which the model must accept +and which must issue each of them, its assertions among them. The two runs need not show +the same outcomes, because the election timeout of a node differs between processes. +Give what went wrong, if anything did. -/ private def roundTrip (directory : System.FilePath) (name : String) : IO String := do let program := directory / s!"{name}.program" @@ -151,7 +153,7 @@ private def roundTrip (directory : System.FilePath) (name : String) : IO String if status != "ok" then return s!"its commands are {status}: {message}" if recorded != ran then - return "its commands show other outcomes" + return "its program issues other commands" return "" /-- diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index 8d6c4d7a164..19c0810ccce 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -16,7 +16,8 @@ build/cft_model_driver --all It replays each command as soon as the driver has run it, and requires each undelivered message of the driver to be a message which the model holds. It delivers the messages which the model holds, newest first, and asserts of the driver the commit indexes of the model. - `--all` runs every scenario and the run in step, each in a process of its own, because a process can start the driver once. - It also runs each scenario again from its [recorded commands](#programs), which must show the same outcomes, and [explores](#exploring) briefly. + It also runs each scenario again from its [recorded commands](#programs), which the model must accept and which must all apply, and [explores](#exploring) briefly. + The two runs of a scenario may differ where an election timeout decides, because the timeout of a node differs between processes. A run prints its status as JSON. It exits with status 1 if a command fails or the model rejects the trace, and 2 if its arguments are wrong. From 890ce59685b574abe1c020b2c9ffd7a7b9e139b2 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 09:47:11 +0000 Subject: [PATCH 19/31] Keep retired nodes out of later consensus scenario configurations trustNode, trustNodes, swapNode and swapNodes replicated a configuration of every node which the driver had created, less those which the command removes. The next such command therefore added back a node which an earlier one had removed. CCF does not do that: a node which has retired is never trusted again. These commands now change the latest configuration which the primary holds. No scenario relied on this: the trace of each has the same commands and configurations as before. A scenario which adds a node after one has retired did, as exploration with joins does. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.h | 41 ++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 22818128117..caaa0b62209 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -289,6 +289,19 @@ class CftDriver _nodes.emplace(node_id, NodeDriver{kv, cft}); } + // The configuration which a reconfiguration on the primary in term changes: + // the latest which that primary holds, as the nodes which a real primary + // trusts are. A node which an earlier configuration removed stays out of it. + ccf::kv::Configuration::Nodes latest_configuration(const std::string& term) + { + const auto primary = find_primary_in_term(term, false); + if (!primary.has_value()) + { + return {}; + } + return _nodes.at(primary->second).cft->get_latest_configuration(); + } + public: CftDriver() = default; @@ -378,10 +391,10 @@ class CftDriver CFT_DRIVER_PRINT( "Note over {}: Node {} trusted", ccf::NodeId(node_id), node_id); } - ccf::kv::Configuration::Nodes configuration; - for (const auto& [id, node] : _nodes) + ccf::kv::Configuration::Nodes configuration = latest_configuration(term); + for (const auto& node_id : node_ids) { - configuration.try_emplace(id); + configuration.try_emplace(node_id); } for (const auto& node_id : node_ids) { @@ -412,15 +425,15 @@ class CftDriver CFT_DRIVER_PRINT( "Note over {}: Node {} retired", ccf::NodeId(node_out), node_out); } - std::set out(nodes_out.begin(), nodes_out.end()); - ccf::kv::Configuration::Nodes configuration; - for (const auto& [id, node] : _nodes) + ccf::kv::Configuration::Nodes configuration = latest_configuration(term); + for (const auto& node_out : nodes_out) { - if (out.find(id) == out.end()) - { - configuration.try_emplace(id); - } + configuration.erase(node_out); + } + for (const auto& node_in : nodes_in) + { + configuration.try_emplace(node_in); } for (const auto& [id, node] : _nodes) { @@ -1103,8 +1116,9 @@ class CftDriver return primaries; } + // Unless required, there may be no primary in the term which term_s names std::optional> find_primary_in_term( - const std::string& term_s) + const std::string& term_s, bool required = true) { auto primaries = find_primaries(); @@ -1134,6 +1148,11 @@ class CftDriver } } + if (!required) + { + return std::nullopt; + } + throw std::runtime_error( fmt::format("Found no primary in term {}", term_s)); } From 57b2a81e2d6ba1f8ef04b874b433e0a0e9b88014 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 09:53:38 +0000 Subject: [PATCH 20/31] Add a consensus scenario command for joining from a snapshot trustNodeFromSnapshot creates a node as trustNode does, but the node starts as a real node does from a snapshot which the primary took at its commit index: at that index, in its term, with the configuration of that index. It also holds the entries before the snapshot, like a node which was given the ledger files of the primary, so that the driver can still compare whole ledgers. The scenario join_from_snapshot shows the outcome joinFromSnapshot of the model, which no trace showed. The scenarios now show 78 of its 79 outcomes. The explorer adds a node at random, empty or from a snapshot, and changes a trust_node of a scenario into one from a snapshot. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lean/consensus/replay/README.md | 2 +- src/consensus/aft/test/driver.cpp | 10 +++ src/consensus/aft/test/driver.h | 84 +++++++++++++++++-- tests/cft_model/ModelDriver/Command.lean | 4 + tests/cft_model/ModelDriver/Episode.lean | 6 ++ tests/cft_model/ModelDriver/Explore.lean | 20 ++++- tests/cft_model/README.md | 2 +- tests/cft_scenarios/CftDriver.lean | 7 ++ tests/cft_scenarios/README.md | 1 + tests/cft_scenarios/Scenarios.lean | 1 + .../Scenarios/join_from_snapshot.lean | 36 ++++++++ 11 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/join_from_snapshot.lean diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index 7ce0ee23bcf..d3171806c0b 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -44,7 +44,7 @@ To report how many steps and traces show each outcome, and which outcomes no tra ``` An outcome which no trace shows is a missing scenario, a step which the driver or the reduction cannot produce, or behaviour which the model allows and the implementation lacks. -For example, the driver starts every node with an empty ledger, so no trace of a scenario shows `joinFromSnapshot`. +For example, no trace of a scenario shows `appendEntriesConflictIgnored`, which [`cft.h`](../../../src/consensus/aft/cft.h) logs as a branch which it expects not to reach. The [consensus model driver](../../../tests/cft_model/README.md#exploring) looks for runs which show such an outcome, by changing the scenarios at random. To compare with the implementation, configure the scenario build with `-DCOVERAGE=ON` and run the scenario test. diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 3c7caea05b9..77721fa93ef 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -472,6 +472,16 @@ extern "C" command("trust_nodes", t, ids), [&]() { driver->trust_nodes(t, ids); }); } + lean_obj_res cft_trust_node_from_snapshot( + b_lean_obj_arg term, b_lean_obj_arg node) + { + const auto t = text(term); + const auto id = text(node); + return step(command("trust_node_from_snapshot", t, id), [&]() { + driver->trust_nodes(t, {id}, true); + }); + } + lean_obj_res cft_swap_node( b_lean_obj_arg term, b_lean_obj_arg retire, b_lean_obj_arg trust) { diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index caaa0b62209..79d62652570 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -244,9 +244,45 @@ class CftDriver return {idx, d, committable, hooks}; } - // A joining node starts as a follower with an empty ledger, like a real - // node joining without a snapshot. - void add_node(ccf::NodeId node_id, bool joining = true) + // What a real node does when it joins from a snapshot which source took at + // the index which it has committed: it adopts the configuration of the + // snapshot, and starts as a follower at that index. It also holds the + // entries before the snapshot, like a real node which was given the + // committed ledger files of source, so that ledgers can still be compared. + static void init_from_snapshot(TCft& cft, TCft& source) + { + const auto idx = source.get_committed_seqno(); + if (idx == 0) + { + cft.init_as_backup(0, 0, {}); + return; + } + + std::optional configuration; + for (const auto& conf : source.get_details().configs) + { + if (conf.idx <= idx) + { + configuration = conf.nodes; + } + } + if (configuration.has_value()) + { + cft.add_configuration(idx, configuration.value()); + } + + const auto& entries = source.ledger->ledger; + cft.ledger->ledger.assign( + entries.begin(), entries.begin() + static_cast(idx)); + cft.init_as_backup(idx, source.get_view(idx), source.get_view_history(idx)); + } + + // A joining node starts as a follower: with an empty ledger, like a real + // node joining without a snapshot, or from a snapshot of another node. + void add_node( + ccf::NodeId node_id, + bool joining = true, + const std::optional& snapshot_of = std::nullopt) { auto kv = std::make_shared(node_id); const ccf::consensus::Configuration settings{{"10ms"}, {"100ms"}}; @@ -276,7 +312,14 @@ class CftDriver }); if (joining) { - cft->init_as_backup(0, 0, {}); + if (snapshot_of.has_value()) + { + init_from_snapshot(*cft, *_nodes.at(snapshot_of.value()).cft); + } + else + { + cft->init_as_backup(0, 0, {}); + } } // The node starts its election timer as a real node does: when it first // holds a signature while it is in its latest configuration. @@ -382,14 +425,39 @@ class CftDriver _replicate(term, {}, false, std::nullopt, retired_committed); } + // With from_snapshot, each new node starts from a snapshot of the primary in + // term. If there is no such primary, it starts empty, as it does without. void trust_nodes( - const std::string& term, const std::vector& node_ids) + const std::string& term, + const std::vector& node_ids, + bool from_snapshot = false) { + std::optional snapshot_of = std::nullopt; + if (from_snapshot) + { + const auto primary = find_primary_in_term(term, false); + if (primary.has_value()) + { + snapshot_of = primary->second; + } + } for (const auto& node_id : node_ids) { - add_node(node_id); - CFT_DRIVER_PRINT( - "Note over {}: Node {} trusted", ccf::NodeId(node_id), node_id); + add_node(node_id, true, snapshot_of); + if (snapshot_of.has_value()) + { + CFT_DRIVER_PRINT( + "Note over {}: Node {} trusted, from a snapshot of {} at {}", + ccf::NodeId(node_id), + node_id, + snapshot_of.value(), + _nodes.at(node_id).cft->get_last_idx()); + } + else + { + CFT_DRIVER_PRINT( + "Note over {}: Node {} trusted", ccf::NodeId(node_id), node_id); + } } ccf::kv::Configuration::Nodes configuration = latest_configuration(term); for (const auto& node_id : node_ids) diff --git a/tests/cft_model/ModelDriver/Command.lean b/tests/cft_model/ModelDriver/Command.lean index 77550a2f374..a9c1c143bd9 100644 --- a/tests/cft_model/ModelDriver/Command.lean +++ b/tests/cft_model/ModelDriver/Command.lean @@ -52,6 +52,7 @@ def run (command : Command) : IO Unit := do | "start_node", [node] => startNode node | "trust_node", [term, node] => trustNode term node | "trust_nodes", term :: nodes => trustNodes term (texts nodes) + | "trust_node_from_snapshot", [term, node] => trustNodeFromSnapshot term node | "swap_node", [term, retire, trust] => swapNode term retire trust | "swap_nodes", term :: items => let (retire, trust) := changes items @@ -126,6 +127,7 @@ def creates (command : Command) : List String := | "start_node", [node] => [node] | "create_new_node", [node] => [node] | "trust_node", [_, node] => [node] + | "trust_node_from_snapshot", [_, node] => [node] | "swap_node", [_, _, node] => [node] | "trust_nodes", _ :: nodes => nodes | "replicate_new_configuration", _ :: nodes => nodes @@ -172,6 +174,7 @@ private def namesTerm : List String := [ "trust_node", "trust_nodes", + "trust_node_from_snapshot", "swap_node", "swap_nodes", "cleanup_nodes", @@ -205,6 +208,7 @@ def scenarioLine (command : Command) : Except String String := match command.name, command.args with | "start_node", [_] => values "startNode" | "trust_node", [_, _] => values "trustNode" + | "trust_node_from_snapshot", [_, _] => values "trustNodeFromSnapshot" | "trust_nodes", term :: nodes => call "trustNodes" [value term, list (nodes.map value)] | "swap_node", [_, _, _] => values "swapNode" | "swap_nodes", term :: items => diff --git a/tests/cft_model/ModelDriver/Episode.lean b/tests/cft_model/ModelDriver/Episode.lean index e470a5bd312..52316b6395a 100644 --- a/tests/cft_model/ModelDriver/Episode.lean +++ b/tests/cft_model/ModelDriver/Episode.lean @@ -365,6 +365,12 @@ private def choose (kind : String) (value : Nat) (nodes isolated : List String) | "write" => return some { name := "replicate", args := ["latest", s!"m{value % 1000}"] } | "sign" => return some { name := "replicate", args := ["latest", "$"] } + | "join" => + -- A node which does not exist yet: one more number than there are nodes has one + let fresh := + ((List.range (nodes.length + 1)).map toString).find? (!nodes.contains ·) + let name := if value % 2 == 0 then "trust_node_from_snapshot" else "trust_node" + return fresh.map fun node => { name, args := ["latest", node] } | _ => fail s!"unknown choice '{kind}'" /-- diff --git a/tests/cft_model/ModelDriver/Explore.lean b/tests/cft_model/ModelDriver/Explore.lean index cdb203c3db3..5825d1ffa19 100644 --- a/tests/cft_model/ModelDriver/Explore.lean +++ b/tests/cft_model/ModelDriver/Explore.lean @@ -55,6 +55,8 @@ private def changes : Array String := ("nominate", 1), ("write", 1), ("sign", 1), + ("join", 1), + ("snapshot", 1), ("duplicate", 1), ("swap", 1), ("latest", 1), @@ -74,7 +76,8 @@ private def choices : Array String := ("rejoin", 1), ("nominate", 1), ("write", 1), - ("sign", 1) + ("sign", 1), + ("join", 1) ] /-- @@ -138,6 +141,21 @@ private def change (steps : Array Step) : Random (Array Step × String) := do if position < size then return (steps.extract 0 position, s!"truncate {position}") return (steps, "") + | "snapshot" => + -- Have a node which joins start from a snapshot of the primary + let joins := + (Array.range size).filter + fun (index : Nat) => + match steps[index]? with + | some (.given command) => command.name == "trust_node" + | _ => false + match joins[value % joins.size]? with + | some index => + let fromSnapshot : Step → Step + | .given command => .given { command with name := "trust_node_from_snapshot" } + | step => step + return (steps.modify index fromSnapshot, s!"snapshot {index}") + | none => return (steps, "") | kind => return (insert steps position (.choice kind value), s!"{kind} {position}") /-- Why a program is run. -/ diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index 19c0810ccce..04c86743a5e 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -50,7 +50,7 @@ build/cft_model_driver --explore [--seed N] [--runs N] [--jobs N] ``` The commands of each scenario, without its assertions, are a first program. -The model driver changes a program at random: it holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, or leaves out, repeats or swaps commands. +The model driver changes a program at random: it holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, adds a node which starts empty or from a snapshot, or leaves out, repeats or swaps commands. It runs each changed program in a process of its own. - A run is kept, to be changed in turn, if the model shows an outcome, or the run reaches an edge of [`cft.h`](../../src/consensus/aft/cft.h), which no run before it has. diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index d40b2229f8f..85e8d8a6ecb 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -46,6 +46,13 @@ opaque trustNode (term : @& Term) (node : @& NodeId) : IO Unit @[extern "cft_trust_nodes"] opaque trustNodes (term : @& Term) (nodes : @& List NodeId) : IO Unit +/-- +As `trustNode`, but `node` starts from a snapshot of the primary in `term`: it holds what +that primary has committed, and the configuration which the primary had then. +-/ +@[extern "cft_trust_node_from_snapshot"] +opaque trustNodeFromSnapshot (term : @& Term) (node : @& NodeId) : IO Unit + /-- Replicate a configuration which removes `retire` and adds the new node `trust`. -/ @[extern "cft_swap_node"] opaque swapNode (term : @& Term) (retire trust : @& NodeId) : IO Unit diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index a7b5a619050..b6a2032fdb6 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -28,6 +28,7 @@ initialize scenario "example" do [`CftDriver.lean`](CftDriver.lean) documents every command. Node identifiers and terms are written as numbers or strings, and `latest` is the primary in the highest term. Each `replicate` appends one batch of entries, which are transaction data or `signature`. +A node which `trustNode` adds starts with an empty ledger, and one which `trustNodeFromSnapshot` adds starts from a snapshot of the primary, at the index which that primary has committed. A scenario delivers the messages of a node in the order in which they were sent, or picks one: `dispatchNth` delivers it, `dropNth` loses it, and `holdNth` sets it aside until `dispatchHeld` delivers it, however late. Scenarios are ordinary Lean, so they can also use loops, conditions and definitions. diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 8437877c852..5952df93b06 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -21,6 +21,7 @@ import Scenarios.follower_overcommit_partial_verified import Scenarios.follower_rollback_match_index import Scenarios.heartbeat_before_stale_suffix import Scenarios.indirect_commit_stale_leader +import Scenarios.join_from_snapshot import Scenarios.large_entry_batching import Scenarios.late_nomination import Scenarios.leadership_race diff --git a/tests/cft_scenarios/Scenarios/join_from_snapshot.lean b/tests/cft_scenarios/Scenarios/join_from_snapshot.lean new file mode 100644 index 00000000000..1c864c100fa --- /dev/null +++ b/tests/cft_scenarios/Scenarios/join_from_snapshot.lean @@ -0,0 +1,36 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node joins from a snapshot of the primary. It starts at the index which the +-- primary has committed, with the configuration of that index, and replicates +-- only what follows it. +initialize scenario "join_from_snapshot" do + startNode 0 + replicate 2 ["a", signature] + assertCommitIdx 0 3 + + -- 2.4 is not committed, so the snapshot ends before it + replicate 2 ["b"] + trustNodeFromSnapshot 2 1 + assertLastTxid 1 "2.3" + assertCommitIdx 1 3 + assertConfig 1 3 [0] + assertDetail 1 "leadership_state" "Follower" + + -- 0 signs at 2.6, after the configuration which adds 1 at 2.5 + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + + assertLastTxid 1 "2.6" + assertConfig 1 5 [0, 1] + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 6 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 6 From 8dccd774195e2b3eba69c7e660ff5d66207568ec Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 11:02:29 +0000 Subject: [PATCH 21/31] Record a new term at the first entry which a follower receives in it After a follower has handled an AppendEntries which does not end in a signature, and whose term is new to it, it recorded that term as starting after its last signature. That is wrong when the follower holds transactions after that signature in an earlier term. It then reported those transactions in the new term. If it had just replaced entries of a later term, the view history refused the record with 'version must not move backwards': recv_message logged the exception, and the follower sent no response and went on reporting the replaced term for the new entry. The follower now records the term as starting at the first entry of the AppendEntries, as it already does when one of its entries is a signature. The two indexes are the same unless a primary kept transactions after its last signature when it was elected. become_leader only allows that if the node has committed nothing, so a running service should not get there. It was found by exploring changes to the scenarios, and unsigned_entry_replaces_stale_suffix fails without this change. In 10,000 explored runs the indexes differ in 544 of 4,617 such records, each in the term of a primary which kept such transactions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/cft.h | 19 ++-- tests/cft_scenarios/Scenarios.lean | 1 + .../unsigned_entry_replaces_stale_suffix.lean | 88 +++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/unsigned_entry_replaces_stale_suffix.lean diff --git a/src/consensus/aft/cft.h b/src/consensus/aft/cft.h index b87059e0fdf..d2b8eb5ceaf 100644 --- a/src/consensus/aft/cft.h +++ b/src/consensus/aft/cft.h @@ -1482,14 +1482,21 @@ namespace aft else { // The end of this append entries (r.idx) was not a signature, but may - // be in a new term. If it's a new term, this term started immediately - // after the previous signature we saw (lci, last committable index). - // If we already know the term of r.idx, there is nothing to record. - // We may then still hold entries of a later term after r.idx, which - // the sender has yet to replace, and that term starts after lci + 1. + // be in a new term. If we already know the term of r.idx, there is + // nothing to record. We may then still hold entries of a later term + // after r.idx, which the sender has yet to replace. + // Otherwise the new term starts at the first entry of this append + // entries, which holds entries of a single term, after an entry which + // we hold in an earlier term. That is immediately after the previous + // signature we saw (lci, last committable index), unless a primary + // kept transactions after its last signature when it was elected, + // which it only does if it has committed nothing. + static_assert( + max_terms_per_append_entries == 1, + "AppendEntries processing for term updates assumes single term"); if (r.idx > lci && get_term_internal(r.idx) != r.term_of_idx) { - state->view_history.update(lci + 1, r.term_of_idx); + state->view_history.update(r.prev_idx + 1, r.term_of_idx); } } diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 5952df93b06..330ff76c49f 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -75,6 +75,7 @@ import Scenarios.suffix_collision_1 import Scenarios.suffix_collision_2 import Scenarios.suffix_collision_3 import Scenarios.swap_single_node +import Scenarios.unsigned_entry_replaces_stale_suffix import Scenarios.vote_request_after_append_entries /-! diff --git a/tests/cft_scenarios/Scenarios/unsigned_entry_replaces_stale_suffix.lean b/tests/cft_scenarios/Scenarios/unsigned_entry_replaces_stale_suffix.lean new file mode 100644 index 00000000000..5fd3347f209 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/unsigned_entry_replaces_stale_suffix.lean @@ -0,0 +1,88 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- A follower replaces an entry of an earlier primary with a transaction of its +-- primary, and acknowledges it. Nothing is committed, so a node which is +-- elected keeps the transactions after its last signature. Node 1 holds 2.4, +-- then 3.5 and 3.6 of its own term, none of them signed, and node 0 is elected +-- in term 4 with 2.4 and 3.5. Its transaction 4.6 replaces 3.6 on node 1: term +-- 4 starts there, and not after 2.3, the last signature of node 1. +-- Found by `cft_model_driver --explore`, from `retirement_signature_rollback`, +-- then written for two nodes. +initialize scenario "unsigned_entry_replaces_stale_suffix" do + startNode 0 + trustNode 2 1 + replicate 2 [signature, "x"] + dispatchAll + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "2.4" + -- Initial network state [0, 1]. The acknowledgements of 1 are lost, so + -- nothing is committed + dropPending 1 + + -- 1 is elected in term 3, and keeps 2.4, which is not signed + periodicOne 1 100 + dispatchOne 1 + dispatchOne 0 + dispatchOne 1 + dispatchOne 0 + assertDetail 1 "leadership_state" "Leader" + assertLastTxid 1 "2.4" + dropPending 1 + + -- 1 appends 3.5, which 0 receives, and 3.6, which it does not + replicate 3 ["y"] + periodicOne 1 10 + dispatchOne 1 + assertLastTxid 0 "3.5" + dropPending 0 + replicate 3 ["z"] + assertLastTxid 1 "3.6" + + -- 0 is elected in term 4, and keeps 2.4 and 3.5 + periodicOne 0 100 + dispatchOne 0 + dispatchOne 1 + dispatchOne 0 + dispatchOne 1 + assertDetail 0 "leadership_state" "Leader" + assertDetail 1 "leadership_state" "Follower" + assertLastTxid 0 "3.5" + + -- The first heartbeat of 0 ends at 3.5. 1 acknowledges it, and keeps 3.6 + dispatchOne 0 + assertLastTxid 1 "3.6" + dispatchOne 1 + + -- 0 appends 4.6, which is not a signature. It replaces 3.6 on 1, which + -- acknowledges it + replicate 4 ["w"] + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "4.6" + assertPending 1 0 1 + dispatchOne 1 + + -- 0 signs at 4.7, and commits + replicate 4 [signature] + periodicOne 0 10 + dispatchOne 0 + assertLastTxid 1 "4.7" + dispatchOne 1 + periodicOne 0 10 + dispatchOne 0 + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 7 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 7 From 1a954ac62639d444cd2dd2bb43fad4b5520f1e09 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 11:55:17 +0000 Subject: [PATCH 22/31] Check what scenario nodes report against their ledgers After each command, the scenario driver now also asserts of every node that: - its view history gives each entry of its ledger the term which the ledger records for it, and it reports its commit with that transaction ID; - what it reports of the primary, of whether it can replicate, and of whether it should sign, agrees with its role and its ledger; - it has notified its commit observer of each advance of its commit index; - if it is primary, it has asked for retired nodes to be cleaned up since it had a retirement to confirm. The trace of a scenario does not show these, so replay against the model cannot check them. The driver now sets the two callbacks of a node, and asserts the same once it has confirmed that the network can make progress. The model driver reports a failure of these as an invariant. The first fails in retirement_signature_rollback without the previous commit: a node reports 4.4 for the entry which its ledger holds as 3.4. Each of them fails in at least two scenarios when cft.h is changed not to do what it checks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.cpp | 1 + src/consensus/aft/test/driver.h | 321 ++++++++++++++++++++++- tests/cft_model/ModelDriver/Episode.lean | 7 +- tests/cft_model/README.md | 2 +- tests/cft_scenarios/CftDriver.lean | 3 +- tests/cft_scenarios/README.md | 11 +- 6 files changed, 337 insertions(+), 8 deletions(-) diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 77721fa93ef..6ae61cf0b19 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -415,6 +415,7 @@ extern "C" return io([]() { // Confirm path to liveness from final state driver->loop_until_sync(); + driver->assert_invariants(); driver.reset(); ccf::tracing::FluentdSink::shutdown(); diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 79d62652570..89b30943c8f 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -133,10 +133,34 @@ aft::ChannelStubProxy* channel_stub_proxy(const TCft& r) class CftDriver { private: + // Records the commits of which a node notifies its observer + struct CommitRecorder : public aft::CommitObserver + { + std::vector notified; + + void on_commit( + ccf::TxID committed, const aft::ViewHistory& /*view_history*/) override + { + notified.push_back(committed); + } + }; + struct NodeDriver { std::shared_ptr kv; std::shared_ptr cft; + std::shared_ptr commits; + // How many times the node has asked for retired nodes to be cleaned up + std::shared_ptr cleanup_requests; + + // What assert_invariants last saw of the node: its commit index, how many + // commits and cleanup requests it had made, the view in which it was + // primary, and the retirements which it then had yet to confirm + aft::Index seen_commit_idx = 0; + size_t seen_commits = 0; + size_t seen_cleanup_requests = 0; + std::optional seen_primary_view = std::nullopt; + std::set seen_unconfirmed = {}; }; // A message which hold_nth has set aside @@ -287,13 +311,16 @@ class CftDriver auto kv = std::make_shared(node_id); const ccf::consensus::Configuration settings{{"10ms"}, {"100ms"}}; auto channels = std::make_shared(); + auto commits = std::make_shared(); + auto cleanup_requests = std::make_shared(0); auto cft = std::make_shared( settings, std::make_unique(kv), std::make_unique(node_id), channels, std::make_shared(node_id), - nullptr); + [cleanup_requests]() { ++*cleanup_requests; }, + commits); channels->read_entries = [ledger = cft->ledger.get()](const aft::AppendEntries& ae) { auto entries = ledger->get_append_entries_payload(ae); @@ -329,7 +356,10 @@ class CftDriver throw std::logic_error(fmt::format("Node {} already exists", node_id)); } - _nodes.emplace(node_id, NodeDriver{kv, cft}); + NodeDriver node{kv, cft, commits, cleanup_requests}; + // A node which starts from a snapshot has committed it, without notice + node.seen_commit_idx = cft->get_committed_seqno(); + _nodes.emplace(node_id, node); } // The configuration which a reconfiguration on the primary in term changes: @@ -345,6 +375,285 @@ class CftDriver return _nodes.at(primary->second).cft->get_latest_configuration(); } + // An entry as the ledger of a node holds it: see LedgerStubProxy::put_entry + struct LedgerEntry + { + bool committable; + aft::Term term; + aft::ReplicatedData data; + }; + + static std::optional ledger_entry(TCft& cft, aft::Index idx) + { + const auto entry = cft.ledger->get_entry_by_idx(idx); + if (!entry.has_value() || entry->empty()) + { + return std::nullopt; + } + + const uint8_t* data = entry->data(); + auto size = entry->size(); + serialized::skip(data, size, sizeof(size_t)); + const auto committable = serialized::read(data, size); + const auto term = serialized::read(data, size); + serialized::skip(data, size, sizeof(ccf::kv::Version)); + return LedgerEntry{ + committable, term, nlohmann::json::parse(std::span{data, size})}; + } + + static std::set named_nodes(const aft::ReplicatedData& data) + { + const ccf::kv::Configuration::Nodes nodes = + nlohmann::json::parse(data.data); + std::set ids; + for (const auto& [id, _] : nodes) + { + ids.insert(id); + } + return ids; + } + + // The nodes whose retirement a node has committed, and which no entry of + // its ledger confirms yet. A real primary confirms the retirement of these + // when it handles the request of its consensus to clean up retired nodes. + static std::set unconfirmed_retirements(TCft& cft) + { + std::set retired; + std::set latest; + std::set confirmed; + const auto commit_idx = cft.get_committed_seqno(); + for (aft::Index idx = 1; idx <= cft.get_last_idx(); ++idx) + { + const auto entry = ledger_entry(cft, idx); + if (!entry.has_value()) + { + continue; + } + + if ( + entry->data.type == aft::ReplicatedDataType::reconfiguration && + idx <= commit_idx) + { + const auto nodes = named_nodes(entry->data); + for (const auto& id : latest) + { + if (!nodes.contains(id)) + { + retired.insert(id); + } + } + latest = nodes; + } + else if (entry->data.type == aft::ReplicatedDataType::retired_committed) + { + confirmed.merge(named_nodes(entry->data)); + } + } + + std::set unconfirmed; + for (const auto& id : retired) + { + if (!latest.contains(id) && !confirmed.contains(id)) + { + unconfirmed.insert(id); + } + } + return unconfirmed; + } + + static std::runtime_error broken(const std::string& what) + { + return std::runtime_error(fmt::format("Invariant broken: {}", what)); + } + + // The view history of a node gives each entry of its ledger the term which + // the ledger records for it, and so its transaction ID. A real node tells + // the status of a transaction from its view history. + static void assert_view_history(const ccf::NodeId& node_id, TCft& cft) + { + for (aft::Index idx = 1; idx <= cft.get_last_idx(); ++idx) + { + const auto entry = ledger_entry(cft, idx); + if (!entry.has_value()) + { + throw broken(fmt::format("Node {} holds no entry at {}", node_id, idx)); + } + + const auto view = cft.get_view(idx); + if (view != entry->term) + { + throw broken(fmt::format( + "Node {} reports {}.{} for the entry which its ledger holds as {}.{}", + node_id, + view, + idx, + entry->term, + idx)); + } + } + + const auto commit_idx = cft.get_committed_seqno(); + const auto [view, seqno] = cft.get_committed_txid(); + const auto committed = ledger_entry(cft, commit_idx); + if ( + seqno != commit_idx || (committed.has_value() && view != committed->term)) + { + throw broken(fmt::format( + "Node {} reports {}.{} as committed, and its commit index is {}", + node_id, + view, + seqno, + commit_idx)); + } + } + + // What a node reports of the primary agrees with its role, and with its + // ledger: a primary should sign until it has signed in its term + void assert_reported_primary(const ccf::NodeId& node_id, TCft& cft) + { + const auto primary = cft.primary(); + const auto is_primary = cft.is_primary(); + if (is_primary != (primary == node_id) || cft.can_replicate() != is_primary) + { + throw broken(fmt::format( + "Node {} {} primary, {} replicate, and reports {} as primary", + node_id, + is_primary ? "is" : "is not", + cft.can_replicate() ? "can" : "cannot", + primary.has_value() ? primary->value() : "no node")); + } + + if (primary.has_value() && primary != node_id) + { + // It learnt of this primary in its own term, which it has had since + const auto it = _nodes.find(primary.value()); + if ( + it == _nodes.end() || !cft.is_backup() || + it->second.cft->get_view() < cft.get_view()) + { + throw broken(fmt::format( + "Node {} reports {} as primary in term {}", + node_id, + primary.value(), + cft.get_view())); + } + } + + using Disposition = ccf::kv::Consensus::SignatureDisposition; + auto expected = Disposition::CANT_REPLICATE; + if (is_primary) + { + expected = Disposition::SHOULD_SIGN; + const auto view = cft.get_view(); + for (aft::Index idx = cft.get_last_idx(); idx > 0; --idx) + { + const auto entry = ledger_entry(cft, idx); + if (!entry.has_value() || entry->term != view) + { + break; + } + if (entry->committable) + { + expected = Disposition::CAN_SIGN; + break; + } + } + } + const auto reported = cft.get_signature_disposition(); + if (reported != expected) + { + const auto name = [](Disposition disposition) { + switch (disposition) + { + case Disposition::CANT_REPLICATE: + return "cannot replicate"; + case Disposition::CAN_SIGN: + return "can sign"; + case Disposition::SHOULD_SIGN: + return "should sign"; + } + return "?"; + }; + throw broken(fmt::format( + "Node {} reports that it {} in term {}, where it {}", + node_id, + name(reported), + cft.get_view(), + name(expected))); + } + } + + // A node notifies its observer of each advance of its commit index, with + // the transaction ID which its ledger holds there + static void assert_commit_notified( + const ccf::NodeId& node_id, NodeDriver& node) + { + const auto commit_idx = node.cft->get_committed_seqno(); + const auto& notified = node.commits->notified; + auto previous = node.seen_commit_idx; + for (auto i = node.seen_commits; i < notified.size(); ++i) + { + if (notified[i].seqno <= previous) + { + throw broken(fmt::format( + "Node {} notified its observer of a commit at {} after one at {}", + node_id, + notified[i].seqno, + previous)); + } + previous = notified[i].seqno; + } + + const auto committed = ledger_entry(*node.cft, commit_idx); + if ( + previous != commit_idx || + (notified.size() > node.seen_commits && + (!committed.has_value() || notified.back().view != committed->term))) + { + throw broken(fmt::format( + "Node {} has committed {}, and last notified its observer of {}", + node_id, + commit_idx, + notified.empty() ? "nothing" : notified.back().to_str())); + } + + node.seen_commit_idx = commit_idx; + node.seen_commits = notified.size(); + } + + // A primary asks for retired nodes to be cleaned up when it is elected, and + // when it commits a configuration. It has then asked since it had a + // retirement to confirm, if it has one. + static void assert_cleanup_requested( + const ccf::NodeId& node_id, NodeDriver& node) + { + const auto requests = *node.cleanup_requests; + std::optional primary_view = std::nullopt; + std::set unconfirmed; + if (node.cft->is_primary()) + { + primary_view = node.cft->get_view(); + unconfirmed = unconfirmed_retirements(*node.cft); + const auto was_primary = node.seen_primary_view == primary_view; + for (const auto& id : unconfirmed) + { + const auto seen = was_primary && node.seen_unconfirmed.contains(id); + if (!seen && requests == node.seen_cleanup_requests) + { + throw broken(fmt::format( + "Node {} is primary and has committed the retirement of {}, " + "without asking for retired nodes to be cleaned up", + node_id, + id)); + } + } + } + + node.seen_primary_view = primary_view; + node.seen_unconfirmed = unconfirmed; + node.seen_cleanup_requests = requests; + } + public: CftDriver() = default; @@ -1592,6 +1901,14 @@ class CftDriver // Check invariants: // Assert commit_index on all nodes is safe assert_commit_safety_all(); + // Assert that what each node reports and asks for agrees with its ledger + for (auto& [node_id, node] : _nodes) + { + assert_view_history(node_id, *node.cft); + assert_reported_primary(node_id, *node.cft); + assert_commit_notified(node_id, node); + assert_cleanup_requested(node_id, node); + } } void assert_commit_idx(ccf::NodeId node_id, const std::string& idx_s) diff --git a/tests/cft_model/ModelDriver/Episode.lean b/tests/cft_model/ModelDriver/Episode.lean index 52316b6395a..7ca033bb519 100644 --- a/tests/cft_model/ModelDriver/Episode.lean +++ b/tests/cft_model/ModelDriver/Episode.lean @@ -123,7 +123,7 @@ private def stopFor (error : String) : Option String := "No held message at position" ] then none - else if has error ["at unsafe commit idx"] then + else if has error ["at unsafe commit idx", "Invariant broken"] then some "invariant" else if has error ["has no nodes"] then -- A primary refuses a configuration with no nodes, which the model disables @@ -236,7 +236,10 @@ private def conclude (stopped : Option Stop) let report : Report := if !failure.isEmpty && !isStuck failure then { - status := "finding", kind := "error", message := failure, command := "finish" + status := "finding" + kind := (stopFor failure).getD "error" + message := failure + command := "finish" } else match session.feed rest with diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index 04c86743a5e..8788de469c7 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -55,7 +55,7 @@ It runs each changed program in a process of its own. - A run is kept, to be changed in turn, if the model shows an outcome, or the run reaches an edge of [`cft.h`](../../src/consensus/aft/cft.h), which no run before it has. It is run a second time first, and is only kept if it does so again. -- A run is a finding if the driver fails, for example because a commit is unsafe, or if the model rejects its trace. +- A run is a finding if the driver fails, for example because a commit is unsafe or a node reports a transaction ID which its ledger does not hold, or if the model rejects its trace. Findings with the same message, numbers aside, are counted together. - A run is neither if the model does not cover it, for example because a primary is asked to remove a node which has not retired, or to remove every node, which CCF refuses. diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 85e8d8a6ecb..0a2f6d3f2cd 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -9,7 +9,8 @@ Each command is implemented by the C function named in its `extern` attribute, i A scenario is an `IO` action built from these commands and registered with `scenario`. After each command that can alter state, the driver asserts that every node's commit index -is safe. After the last command, it confirms that the network can still make progress. +is safe, and that what each node reports and asks for agrees with its ledger. After the last +command, it confirms that the network can still make progress. -/ namespace CftDriver diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index b6a2032fdb6..1b264421d52 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -32,8 +32,15 @@ A node which `trustNode` adds starts with an empty ledger, and one which `trustN A scenario delivers the messages of a node in the order in which they were sent, or picks one: `dispatchNth` delivers it, `dropNth` loses it, and `holdNth` sets it aside until `dispatchHeld` delivers it, however late. Scenarios are ordinary Lean, so they can also use loops, conditions and definitions. -After each command which can alter state, the driver asserts that the commit index of every node is safe. -After the last command, it confirms that the network can still make progress. +After each command which can alter state, the driver asserts of every node that: + +- a quorum of each of its configurations holds what it has committed; +- its view history gives each entry of its ledger the term which the ledger records for it, and so its transaction ID; +- what it reports of the primary, and of whether it should sign, agrees with its role and its ledger; +- it has notified its commit observer of each advance of its commit index; +- if it is primary, it has asked for retired nodes to be cleaned up since it had a retirement to confirm. + +After the last command, it confirms that the network can still make progress, and asserts these again. A node starts its election timer as a real node does, when it first holds a signature while it is in its latest configuration. Advancing its clock before that starts no election. From b4861b061d739f51d3f4ff4ffa4bdc49c699eb0b Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 12:03:17 +0000 Subject: [PATCH 23/31] Confirm retirements in consensus scenarios as a primary does cleanupRetired replicates what a real primary does when its consensus asks it to clean up retired nodes: it confirms the retirement of each node whose retirement that primary has committed, and has yet to confirm. cleanupNodes still confirms the nodes which a scenario names. In retirement_confirmed_by_new_primary, a primary commits a retirement and loses its role before it confirms it, and the node which is elected confirms it. The explorer can now retire a member of the latest configuration, and confirm the retirements which the primary has committed, wherever it changes a scenario. With the choice of adding a node, it can change the membership of any scenario. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.cpp | 7 ++ src/consensus/aft/test/driver.h | 30 ++++++++ tests/cft_model/ModelDriver/Command.lean | 20 +++++ tests/cft_model/ModelDriver/Episode.lean | 19 ++++- tests/cft_model/ModelDriver/Explore.lean | 6 +- tests/cft_model/README.md | 2 +- tests/cft_scenarios/CftDriver.lean | 9 +++ tests/cft_scenarios/README.md | 1 + tests/cft_scenarios/Scenarios.lean | 1 + .../retirement_confirmed_by_new_primary.lean | 75 +++++++++++++++++++ 10 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/retirement_confirmed_by_new_primary.lean diff --git a/src/consensus/aft/test/driver.cpp b/src/consensus/aft/test/driver.cpp index 6ae61cf0b19..ec97e0166e3 100644 --- a/src/consensus/aft/test/driver.cpp +++ b/src/consensus/aft/test/driver.cpp @@ -525,6 +525,13 @@ extern "C" }); } + lean_obj_res cft_cleanup_retired(b_lean_obj_arg term) + { + const auto t = text(term); + return step( + command("cleanup_retired", t), [&]() { driver->cleanup_retired(t); }); + } + lean_obj_res cft_nominate_successor(b_lean_obj_arg node) { const auto id = text(node); diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 89b30943c8f..2e7118121cd 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -734,6 +734,36 @@ class CftDriver _replicate(term, {}, false, std::nullopt, retired_committed); } + // What a real primary does when it handles the request of its consensus to + // clean up retired nodes: it confirms the retirement of each node whose + // retirement it has committed, and which it has yet to confirm + void cleanup_retired(const std::string& term) + { + const auto primary = find_primary_in_term(term); + if (!primary.has_value()) + { + CFT_DRIVER_PRINT( + "Note left of {}: No primary to clean up retired nodes", + _nodes.begin()->first); + return; + } + + const auto& node_id = primary->second; + const auto unconfirmed = unconfirmed_retirements(*_nodes.at(node_id).cft); + if (unconfirmed.empty()) + { + CFT_DRIVER_PRINT("Note over {}: No retired node to clean up", node_id); + return; + } + + ccf::kv::Configuration::Nodes retired_committed; + for (const auto& id : unconfirmed) + { + retired_committed.try_emplace(id); + } + _replicate(term, {}, false, std::nullopt, retired_committed); + } + // With from_snapshot, each new node starts from a snapshot of the primary in // term. If there is no such primary, it starts empty, as it does without. void trust_nodes( diff --git a/tests/cft_model/ModelDriver/Command.lean b/tests/cft_model/ModelDriver/Command.lean index a9c1c143bd9..e61ca8589d8 100644 --- a/tests/cft_model/ModelDriver/Command.lean +++ b/tests/cft_model/ModelDriver/Command.lean @@ -58,6 +58,7 @@ def run (command : Command) : IO Unit := do let (retire, trust) := changes items swapNodes term (texts retire) (texts trust) | "cleanup_nodes", term :: nodes => cleanupNodes term (texts nodes) + | "cleanup_retired", [term] => cleanupRetired term | "nominate_successor", [node] => nominateSuccessor node | "nodes", nodes => CftDriver.nodes (texts nodes) | "create_new_node", [node] => createNewNode node @@ -135,6 +136,23 @@ def creates (command : Command) : List String := | "swap_nodes", _ :: items => (changes items).2 | _, _ => [] +/-- +The nodes of the latest configuration once the command is issued, where `members` are +those before it. +-/ +def configures (command : Command) (members : List String) : List String := + match command.name, command.args with + | "start_node", [node] => [node] + | "nodes", nodes => nodes + | "replicate_new_configuration", _ :: nodes => nodes + | "trust_node", [_, node] | "trust_node_from_snapshot", [_, node] => members ++ [node] + | "trust_nodes", _ :: nodes => members ++ nodes + | "swap_node", [_, retire, trust] => members.filter (· != retire) ++ [trust] + | "swap_nodes", _ :: items => + let (retire, trust) := changes items + members.filter (!retire.contains ·) ++ trust + | _, _ => members + /-- The commands which the driver cannot run unless the node which they name first exists. -/ private def namesNode : List String := [ @@ -178,6 +196,7 @@ private def namesTerm : List String := "swap_node", "swap_nodes", "cleanup_nodes", + "cleanup_retired", "replicate_new_configuration", "replicate" ] @@ -216,6 +235,7 @@ def scenarioLine (command : Command) : Except String String := call "swapNodes" [value term, list (retire.map value), list (trust.map value)] | "cleanup_nodes", term :: nodes => call "cleanupNodes" [value term, list (nodes.map value)] + | "cleanup_retired", [_] => values "cleanupRetired" | "nominate_successor", [_] => values "nominateSuccessor" | "replicate", term :: entries => let entries := diff --git a/tests/cft_model/ModelDriver/Episode.lean b/tests/cft_model/ModelDriver/Episode.lean index 7ca033bb519..005c1833231 100644 --- a/tests/cft_model/ModelDriver/Episode.lean +++ b/tests/cft_model/ModelDriver/Episode.lean @@ -332,9 +332,10 @@ private def queuedMessages (reported : Array String) : IO (Array Queued) := do /-- The command for a choice, if the state of the driver allows one. `nodes` are those which -exist, and `isolated` those which a command has disconnected from every other. +exist, `members` those which the commands issued leave in the latest configuration, if +none was lost, and `isolated` those which a command has disconnected from every other. -/ -private def choose (kind : String) (value : Nat) (nodes isolated : List String) +private def choose (kind : String) (value : Nat) (nodes members isolated : List String) : IO (Option Command) := do let pick (items : List String) : Option String := items[value % items.length]? let named (name : String) (node : Option String) : Option Command := @@ -374,6 +375,13 @@ private def choose (kind : String) (value : Nat) (nodes isolated : List String) ((List.range (nodes.length + 1)).map toString).find? (!nodes.contains ·) let name := if value % 2 == 0 then "trust_node_from_snapshot" else "trust_node" return fresh.map fun node => { name, args := ["latest", node] } + | "retire" => + -- Not the last member: a primary refuses a configuration with no nodes + if members.length < 2 then + return none + return (pick members).map + fun node => { name := "swap_nodes", args := ["latest", "out", node] } + | "cleanup" => return some { name := "cleanup_retired", args := ["latest"] } | _ => fail s!"unknown choice '{kind}'" /-- @@ -386,13 +394,14 @@ private def execute (steps : Array Step) startCapture let mut issued : Array Command := #[] let mut nodes : List String := [] + let mut members : List String := [] let mut isolated : List String := [] let mut stopped : Option Stop := none for step in steps do let chosen ← match step with | .given command => pure (some command) - | .choice kind value => choose kind value nodes isolated + | .choice kind value => choose kind value nodes members isolated let some command := chosen | continue -- The driver fails these, some after it has created a node if command.needs.any (!nodes.contains ·) then @@ -415,7 +424,9 @@ private def execute (steps : Array Step) catch error => pure (some (toString error)) match failure with - | none => issued := issued.push command + | none => + issued := issued.push command + members := command.configures members | some message => match stopFor message with | none => diff --git a/tests/cft_model/ModelDriver/Explore.lean b/tests/cft_model/ModelDriver/Explore.lean index 5825d1ffa19..e6eb846af59 100644 --- a/tests/cft_model/ModelDriver/Explore.lean +++ b/tests/cft_model/ModelDriver/Explore.lean @@ -57,6 +57,8 @@ private def changes : Array String := ("sign", 1), ("join", 1), ("snapshot", 1), + ("retire", 1), + ("cleanup", 1), ("duplicate", 1), ("swap", 1), ("latest", 1), @@ -77,7 +79,9 @@ private def choices : Array String := ("nominate", 1), ("write", 1), ("sign", 1), - ("join", 1) + ("join", 1), + ("retire", 1), + ("cleanup", 1) ] /-- diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index 8788de469c7..c064707ceff 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -50,7 +50,7 @@ build/cft_model_driver --explore [--seed N] [--runs N] [--jobs N] ``` The commands of each scenario, without its assertions, are a first program. -The model driver changes a program at random: it holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, adds a node which starts empty or from a snapshot, or leaves out, repeats or swaps commands. +The model driver changes a program at random: it holds a message until later, loses or delivers one early, advances the clock of a node, disconnects a node for a while, nominates, replicates, adds a node which starts empty or from a snapshot, retires a node, confirms the retirements which the primary has committed, or leaves out, repeats or swaps commands. It runs each changed program in a process of its own. - A run is kept, to be changed in turn, if the model shows an outcome, or the run reaches an edge of [`cft.h`](../../src/consensus/aft/cft.h), which no run before it has. diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 0a2f6d3f2cd..149569d0e6b 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -67,6 +67,15 @@ opaque swapNodes (term : @& Term) (retire : @& List NodeId := []) @[extern "cft_cleanup_nodes"] opaque cleanupNodes (term : @& Term) (nodes : @& List NodeId) : IO Unit +/-- +Replicate that the retirement is committed of each node whose retirement the primary in +`term` has committed, and has yet to confirm. A real primary does so when its consensus +asks it to clean up retired nodes: when it is elected, and when it commits a configuration. +Does nothing if there is no such node. +-/ +@[extern "cft_cleanup_retired"] +opaque cleanupRetired (term : @& Term) : IO Unit + /-- Ask `node` to nominate a successor. -/ @[extern "cft_nominate_successor"] opaque nominateSuccessor (node : @& NodeId) : IO Unit diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 1b264421d52..682e1578ec9 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -29,6 +29,7 @@ initialize scenario "example" do Node identifiers and terms are written as numbers or strings, and `latest` is the primary in the highest term. Each `replicate` appends one batch of entries, which are transaction data or `signature`. A node which `trustNode` adds starts with an empty ledger, and one which `trustNodeFromSnapshot` adds starts from a snapshot of the primary, at the index which that primary has committed. +`cleanupNodes` confirms the retirement of the nodes which it names, and `cleanupRetired` that of each node whose retirement the primary has committed, as a real primary does when its consensus asks it to clean up retired nodes. A scenario delivers the messages of a node in the order in which they were sent, or picks one: `dispatchNth` delivers it, `dropNth` loses it, and `holdNth` sets it aside until `dispatchHeld` delivers it, however late. Scenarios are ordinary Lean, so they can also use loops, conditions and definitions. diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 330ff76c49f..a803e345bd5 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -57,6 +57,7 @@ import Scenarios.retire_one import Scenarios.retired_committed_delayed_nomination import Scenarios.retired_node_delayed_responses import Scenarios.retirement_commit_below_signature +import Scenarios.retirement_confirmed_by_new_primary import Scenarios.retirement_forgets_lower_node import Scenarios.retirement_nominates_higher_node import Scenarios.retirement_nominates_most_replicated diff --git a/tests/cft_scenarios/Scenarios/retirement_confirmed_by_new_primary.lean b/tests/cft_scenarios/Scenarios/retirement_confirmed_by_new_primary.lean new file mode 100644 index 00000000000..8ebdb78c1ba --- /dev/null +++ b/tests/cft_scenarios/Scenarios/retirement_confirmed_by_new_primary.lean @@ -0,0 +1,75 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- A node which is elected confirms the retirement which the primary before it +-- committed, and did not confirm. Consensus asks each primary to clean up +-- retired nodes: node 0 when it commits the configuration which removes node +-- 2, and node 1 when it is elected. `cleanupRetired` is what a primary then +-- does. Once the confirmation is committed, no node sends to node 2. +initialize scenario "retirement_confirmed_by_new_primary" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + -- Initial network state [0, 1, 2] + assertCommitIdx 0 3 + assertCommitIdx 1 3 + assertCommitIdx 2 3 + + -- 2 retires, and every node commits its retirement + swapNodes 2 (retire := [2]) + replicate 2 [signature] + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 5 + assertCommitIdx 1 5 + assertDetail 2 "retirement_phase" "Completed" + + -- 0 has not confirmed the retirement when 1 is elected in term 3 + periodicOne 1 100 + dispatchOne 1 + dispatchOne 0 + dispatchOne 1 + dispatchOne 0 + assertDetail 1 "leadership_state" "Leader" + assertDetail 0 "leadership_state" "Follower" + dispatchAll + + -- 1 confirms the retirement of 2 at 3.6, and commits it at 3.7 + cleanupRetired 3 + assertLastTxid 1 "3.6" + replicate 3 [signature] + periodicOne 1 10 + assertPending 1 2 1 + dispatchAll + assertCommitIdx 1 7 + periodicOne 1 10 + dispatchAll + assertCommitIdx 0 7 + + -- Neither 0 nor 1 knows 2 any longer, which has nothing left to confirm + periodicOne 1 10 + assertPending 1 2 0 + dispatchAll + cleanupRetired 3 + assertLastTxid 1 "3.7" + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 7 + assertDetail 1 "leadership_state" "Leader" + assertCommitIdx 1 7 From 1a3cbb1a52b8bdfaae1edfce6e14ed859c5e718a Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 15:54:39 +0000 Subject: [PATCH 24/31] Wait only for nodes which the primary knows in consensus scenarios After the last command of a scenario, the driver confirms that the network can still make progress: it signs on the latest primary, and runs until the active nodes hold the same ledger. A node which that primary does not know is sent nothing, and can never catch up. The driver now leaves such a node out, as it leaves out a node which has retired. It still waits for a node which has retired without learning of it, which the primary knows until the retirement is confirmed. No scenario changes. In an exploration of 20,000 changed scenarios, 398 runs ended without this confirmation because the nodes did not agree. With this change and the three which follow, 19 do. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/consensus/aft/test/driver.h | 19 ++++++++++++++----- tests/cft_scenarios/CftDriver.lean | 5 ++++- tests/cft_scenarios/README.md | 1 + 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h index 2e7118121cd..6bf2b3eb3e8 100644 --- a/src/consensus/aft/test/driver.h +++ b/src/consensus/aft/test/driver.h @@ -1804,18 +1804,27 @@ class CftDriver const auto& [term, primary] = term_primary; - // Don't try to confirm sync with retired nodes + // Don't try to confirm sync with retired nodes, or with nodes which the + // primary does not know, and so sends nothing to + const auto known = _nodes.at(primary).cft->get_details().acks; decltype(_nodes) nodes; for (auto& [node_id, node_driver] : _nodes) { - if (node_driver.cft->is_active()) + if (!node_driver.cft->is_active()) { - nodes[node_id] = node_driver; + CFT_DRIVER_PRINT( + "Note over {}: Ignoring from sync check, due to retirement", node_id); } - else + else if (node_id != primary && !known.contains(node_id)) { CFT_DRIVER_PRINT( - "Note over {}: Ignoring from sync check, due to retirement", node_id); + "Note over {}: Ignoring from sync check, as {} does not know it", + node_id, + primary); + } + else + { + nodes[node_id] = node_driver; } } diff --git a/tests/cft_scenarios/CftDriver.lean b/tests/cft_scenarios/CftDriver.lean index 149569d0e6b..fa21f332603 100644 --- a/tests/cft_scenarios/CftDriver.lean +++ b/tests/cft_scenarios/CftDriver.lean @@ -269,7 +269,10 @@ opaque assertAbsentConfig (node : @& NodeId) (idx : UInt64) : IO Unit @[extern "cft_assert_last_txid"] opaque assertLastTxid (node : @& NodeId) (txid : @& String) : IO Unit -/-- Sign on the latest primary, reconnect the active nodes and run until they are in sync. -/ +/-- +Sign on the latest primary, reconnect the active nodes which it knows, and run until they +are in sync. +-/ @[extern "cft_loop_until_sync"] opaque loopUntilSync : IO Unit diff --git a/tests/cft_scenarios/README.md b/tests/cft_scenarios/README.md index 682e1578ec9..a26fe52f4f8 100644 --- a/tests/cft_scenarios/README.md +++ b/tests/cft_scenarios/README.md @@ -42,6 +42,7 @@ After each command which can alter state, the driver asserts of every node that: - if it is primary, it has asked for retired nodes to be cleaned up since it had a retirement to confirm. After the last command, it confirms that the network can still make progress, and asserts these again. +To do so, it signs on the latest primary, and runs until the active nodes which that primary knows are in sync with it. A node starts its election timer as a real node does, when it first holds a signature while it is in its latest configuration. Advancing its clock before that starts no election. From d29a76ad2d63146070bb1e5265c69ccecabf90c2 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 15:55:58 +0000 Subject: [PATCH 25/31] Forget nodes which only a rolled back configuration added A node keeps the nodes to which a primary sends in all_other_nodes. It added a node when a configuration named it, and removed one only once its retirement or its deletion was committed. A rollback removed a configuration, and not the nodes which that configuration had added. A node which is elected discards what follows its last signature. If that includes a configuration which adds a node, the new primary went on sending AppendEntries to a node which none of its configurations named, listed it in /node/consensus, and could nominate it as its successor. Quorums and votes count the configurations only, so what was committed stayed safe. A rollback now forgets the nodes which only the configurations rolled back named. A node whose retirement is not confirmed stays known: retirement is final, so no later configuration names it. The consensus model reads the peers of a node from its log, and cft_model_driver --explore found the difference in 152 of 20,000 changed scenarios. The scenario rollback_forgets_added_node and a unit test fail without the change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .changes/forget-rolled-back-nodes.fixed.md | 2 + doc/architecture/consensus/index.rst | 1 + lean/consensus/Consensus/Model/Node.lean | 3 +- src/consensus/aft/cft.h | 23 +++++++ src/consensus/aft/test/main.cpp | 65 +++++++++++++++++++ tests/cft_scenarios/Scenarios.lean | 1 + .../rollback_forgets_added_node.lean | 54 +++++++++++++++ 7 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 .changes/forget-rolled-back-nodes.fixed.md create mode 100644 tests/cft_scenarios/Scenarios/rollback_forgets_added_node.lean diff --git a/.changes/forget-rolled-back-nodes.fixed.md b/.changes/forget-rolled-back-nodes.fixed.md new file mode 100644 index 00000000000..b83af0d6e46 --- /dev/null +++ b/.changes/forget-rolled-back-nodes.fixed.md @@ -0,0 +1,2 @@ +A node which rolls back an uncommitted reconfiguration transaction now stops tracking the nodes which only that transaction added, where a primary previously continued to send them `AppendEntries` and to list them in `GET /node/consensus`. +See [Reconfiguration](doc/architecture/consensus/index.rst#reconfiguration). diff --git a/doc/architecture/consensus/index.rst b/doc/architecture/consensus/index.rst index 907db8efa30..2722d45d2af 100644 --- a/doc/architecture/consensus/index.rst +++ b/doc/architecture/consensus/index.rst @@ -84,6 +84,7 @@ For a live service, reconfiguration is materialised in two separate transactions In contrast to normal transactions, reconfiguration transactions will only commit when the necessary quorum of acknowledgements is reached in **both** the previous and the new configuration it defines. Transactions before an uncommitted reconfiguration transaction do not need acknowledgements from the new configuration it defines: they commit once their signature has a quorum in each active configuration defined at or before that signature. +A node which rolls back an uncommitted reconfiguration transaction stops tracking the nodes which only that transaction added: they are no longer sent ``AppendEntries`` and no longer appear in :http:GET:`/node/consensus`. Disaster recovery is an exception to this retirement sequence. Replaying the public ledger reconstructs the previous service's trusted-node configuration in the new consensus instance. diff --git a/lean/consensus/Consensus/Model/Node.lean b/lean/consensus/Consensus/Model/Node.lean index 270ff7f591f..00fa1ae6526 100644 --- a/lean/consensus/Consensus/Model/Node.lean +++ b/lean/consensus/Consensus/Model/Node.lean @@ -54,7 +54,8 @@ inductive Event where /-! ## Primary state -/ /-- -Approximates `all_other_nodes`; unlike CCF, peers lost on rollback are not retained. +Approximates `all_other_nodes`; unlike CCF, a node which starts from a snapshot knows the +nodes which retired before it. -/ def eligiblePeers (state : NodeState) (self : NodeID) : Finset NodeID := (state.activeMembers ∪ state.retirementCompletedNodes).erase self diff --git a/src/consensus/aft/cft.h b/src/consensus/aft/cft.h index d2b8eb5ceaf..5ba47597420 100644 --- a/src/consensus/aft/cft.h +++ b/src/consensus/aft/cft.h @@ -2709,18 +2709,41 @@ namespace aft // Rollback configurations. bool changed = false; + std::set rolled_back_nodes; while (!configurations.empty() && (configurations.back().idx > idx)) { CFT_DEBUG_FMT( "Configurations: rollback configuration at {}", configurations.back().idx); + for (const auto& [node_id, _] : configurations.back().nodes) + { + rolled_back_nodes.insert(node_id); + } configurations.pop_back(); changed = true; } if (changed) { + // Forget the nodes which only the configurations rolled back named: + // no configuration is left to make them known. A node which retired + // from an earlier configuration is never named again, so it is not + // among them, and stays known until its retirement is confirmed. + const auto remaining_nodes = other_nodes_in_active_configs(); + for (const auto& node_id : rolled_back_nodes) + { + if ( + !remaining_nodes.contains(node_id) && + all_other_nodes.erase(node_id) != 0) + { + CFT_INFO_FMT( + "Removed {} from nodes known to consensus: the configuration " + "which added it was rolled back", + node_id); + } + } + create_and_remove_node_state(); } } diff --git a/src/consensus/aft/test/main.cpp b/src/consensus/aft/test/main.cpp index 84bdbc3ae96..b9b2fdbe466 100644 --- a/src/consensus/aft/test/main.cpp +++ b/src/consensus/aft/test/main.cpp @@ -1870,6 +1870,71 @@ DOCTEST_TEST_CASE( DOCTEST_REQUIRE(r0.is_primary()); } +DOCTEST_TEST_CASE( + "Nodes added by a rolled back configuration are forgotten" * + doctest::test_suite("multiple")) +{ + ccf::NodeId node_id0 = ccf::kv::test::PrimaryNodeId; + ccf::NodeId node_id1 = ccf::kv::test::FirstBackupNodeId; + ccf::NodeId node_id2 = ccf::kv::test::SecondBackupNodeId; + ccf::NodeId node_id3 = ccf::kv::test::ThirdBackupNodeId; + + TestNode node0(node_id0); + auto& r0 = node0.cft; + auto r0c = channel_stub_proxy(r0); + + aft::Configuration::Nodes config; + config[node_id0] = {}; + config[node_id1] = {}; + config[node_id2] = {}; + r0.add_configuration(0, config); + r0.force_become_primary(); + + DOCTEST_INFO( + "Node 2 retires: its retirement is committed, and not confirmed"); + config.erase(node_id2); + auto retire_hooks = std::make_shared(); + retire_hooks->push_back( + std::make_unique(config, 1)); + auto data = std::make_shared>(3, 1); + DOCTEST_REQUIRE(r0.replicate( + ccf::kv::BatchVector{{1, data, true, retire_hooks}}, r0.get_view())); + const auto ack = as_bytes(aft::AppendEntriesResponse{ + .term = r0.get_view(), + .last_log_idx = 1, + .success = aft::AppendEntriesResponseType::OK}); + r0.recv_message(node_id1, ack); + DOCTEST_REQUIRE(r0.get_committed_seqno() == 1); + DOCTEST_REQUIRE(r0.get_details().configs.size() == 1); + + DOCTEST_INFO("A configuration which is not committed adds node 3"); + config[node_id3] = {}; + auto trust_hooks = std::make_shared(); + trust_hooks->push_back( + std::make_unique(config, 2)); + DOCTEST_REQUIRE(r0.replicate( + ccf::kv::BatchVector{{2, data, true, trust_hooks}}, r0.get_view())); + DOCTEST_REQUIRE(r0.get_details().configs.size() == 2); + DOCTEST_REQUIRE(r0.get_details().acks.contains(node_id3)); + + DOCTEST_INFO("Once it is rolled back, no configuration names node 3"); + r0.rollback(1); + DOCTEST_REQUIRE(r0.get_details().configs.size() == 1); + DOCTEST_REQUIRE_FALSE(r0.get_details().acks.contains(node_id3)); + + DOCTEST_INFO( + "Node 1 is in the configuration which remains, and the retirement of " + "node 2 is still to confirm"); + DOCTEST_REQUIRE(r0.get_details().acks.contains(node_id1)); + DOCTEST_REQUIRE(r0.get_details().acks.contains(node_id2)); + r0c->messages.clear(); + r0.periodic(request_timeout); + DOCTEST_REQUIRE_FALSE( + r0c->pop_first(aft::append_entries, node_id3).has_value()); + DOCTEST_REQUIRE(r0c->pop_first(aft::append_entries, node_id1).has_value()); + DOCTEST_REQUIRE(r0c->pop_first(aft::append_entries, node_id2).has_value()); +} + DOCTEST_TEST_CASE( "cft messages that cannot be parsed are ignored" * doctest::test_suite("multiple")) diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index a803e345bd5..4bb26079de5 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -63,6 +63,7 @@ import Scenarios.retirement_nominates_higher_node import Scenarios.retirement_nominates_most_replicated import Scenarios.retirement_rollback import Scenarios.retirement_signature_rollback +import Scenarios.rollback_forgets_added_node import Scenarios.rollback_safety import Scenarios.soft_rollback import Scenarios.sole_member_election diff --git a/tests/cft_scenarios/Scenarios/rollback_forgets_added_node.lean b/tests/cft_scenarios/Scenarios/rollback_forgets_added_node.lean new file mode 100644 index 00000000000..a381b3ccf73 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/rollback_forgets_added_node.lean @@ -0,0 +1,54 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- A node forgets the node which a configuration added, when it rolls that +-- configuration back. Node 0 replaces itself with node 2 in a configuration +-- which no signature follows. It hears nothing from node 2, and needs its vote +-- to be elected again. Once elected, it discards what follows its last +-- signature, and the configuration with it. No configuration of node 0 then +-- names node 2, so node 0 sends it nothing more. +-- Found by `cft_model_driver --explore`. +initialize scenario "rollback_forgets_added_node" do + startNode 0 + replicate 2 [signature] + assertCommitIdx 0 2 + + -- 0 retires and 2 joins at 2.3, which nothing signs + swapNodes 2 [0] [2] + assertDetail 0 "membership_state" "Retired" + assertConfig 0 3 [2] + + -- 0 stands down for want of an acknowledgement, and calls an election + periodicOne 0 100 + assertDetail 0 "leadership_state" "Follower" + periodicOne 0 150 + assertDetail 0 "leadership_state" "PreVoteCandidate" + + -- 2 rejects the entries, which do not follow its empty ledger, and votes for 0 + dispatchAll + assertDetail 0 "leadership_state" "Leader" + assertDetail 0 "current_view" 3 + assertLastTxid 2 "0.0" + + -- 0 rolled back to 2.2: it is active again, and the configuration is gone + assertLastTxid 0 "2.2" + assertDetail 0 "membership_state" "Active" + assertAbsentConfig 0 3 + + -- 0 no longer knows 2, and sends it neither the signature nor a heartbeat + replicate 3 [signature] + assertCommitIdx 0 3 + periodicOne 0 10 + assertPending 0 2 0 + + assertDetail 2 "leadership_state" "Follower" + assertCommitIdx 2 0 From f951558e024bc3f925cc1ff294294952e3310469 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 15:57:55 +0000 Subject: [PATCH 26/31] Replay a primary which does not know nodes retired before its snapshot A node which joins from a snapshot is given the configuration at that snapshot. Once elected, it sends neither entries nor a nomination to a node which retired before the snapshot and whose retirement is not confirmed. Every other node still knows that node, and so does the consensus model, which reads the peers of a node from its whole log. Replay now records the index at which each node starts, and treats what the model sends to such a node as lost, which the model allows. Only entries and nominations are excused, and only from a node to a node which a configuration before its snapshot had removed. cft_model_driver --explore found the difference in 3 of 20,000 changed scenarios. The scenario join_from_snapshot_after_retirement shows it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lean/consensus/Consensus.lean | 1 + .../Consensus/Replay/Interpreter.lean | 31 +++++- .../Consensus/Tests/ReplayDifferences.lean | 96 +++++++++++++++++++ lean/consensus/README.md | 1 + lean/consensus/replay/README.md | 5 + tests/cft_scenarios/Scenarios.lean | 1 + .../join_from_snapshot_after_retirement.lean | 88 +++++++++++++++++ 7 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 lean/consensus/Consensus/Tests/ReplayDifferences.lean create mode 100644 tests/cft_scenarios/Scenarios/join_from_snapshot_after_retirement.lean diff --git a/lean/consensus/Consensus.lean b/lean/consensus/Consensus.lean index 6caccb98374..5bfe07c2525 100644 --- a/lean/consensus/Consensus.lean +++ b/lean/consensus/Consensus.lean @@ -5,5 +5,6 @@ public import Consensus.Replay.Session import Consensus.Tests.CanonicalTraces import Consensus.Tests.ReplayDelivery +import Consensus.Tests.ReplayDifferences import Consensus.Tests.ReplayOutcomes import Consensus.Tests.ReplaySession diff --git a/lean/consensus/Consensus/Replay/Interpreter.lean b/lean/consensus/Consensus/Replay/Interpreter.lean index c77e9981f8b..7760d038f45 100644 --- a/lean/consensus/Consensus/Replay/Interpreter.lean +++ b/lean/consensus/Consensus/Replay/Interpreter.lean @@ -65,6 +65,8 @@ structure ReplayState where queued : List Envelope := [] /-- Outcomes shown by the actions run so far, in order. -/ shown : Array Outcome := #[] + /-- The index of the snapshot from which each joining node started, zero if it had none. -/ + snapshots : List (NodeID × Nat) := [] /-- A traced native send, or the proposal a nomination implies. -/ structure NativeSend where @@ -287,11 +289,26 @@ private def ReplayState.run (replay : ReplayState) (header : Header) -- A step appends the acting node's sends to the network left after any delivery. let pending := replay.state.network.length - if action matches .deliver _ then 1 else 0 return { - state, - queued := replay.queued ++ state.network.drop pending, - shown := replay.shown ++ outcomes replay.state action + replay with + state, + queued := replay.queued ++ state.network.drop pending, + shown := replay.shown ++ outcomes replay.state action } +/-- +Whether CCF cannot send `envelope`: entries or a nomination, from a node to a peer which it +has never known because that peer retired before the snapshot from which the node started. +CCF gives such a node the configuration at its snapshot. The model reads the peers of a +node from its whole log, and sends to that peer until its retirement is confirmed. +-/ +private def toUnknownPeer (replay : ReplayState) (envelope : Envelope) : Bool := + match envelope.payload, replay.snapshots.lookup envelope.source, + nodeState replay.state envelope.source with + | .appendEntriesRequest _, some index, some state + | .proposeRequestVoteRequest _, some index, some state => + decide (envelope.target ∈ removedNodes (state.log.take index)) + | _, _, _ => false + /-- Observations never change model state or repair a disabled action. -/ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instruction) : Except String ReplayState := do @@ -308,7 +325,8 @@ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instr | throw s!"{context}: not queued by the model: {send.packet.compress}" (checkPacket send.packet envelope.payload).mapError (s!"{context}: " ++ ·) queued := removeOne envelope queued - if let some envelope := queued.head? then + -- What the model sends to a peer which CCF does not know is lost + if let some envelope := queued.find? (!toUnknownPeer replay ·) then let target := header.declared[envelope.target]?.getD (toString envelope.target) throw s!"model queued {(messageJson envelope.payload).compress} to '{target}' with no native send" @@ -347,7 +365,10 @@ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instr && (index == 0 || isSignatureAt state.log index) && termAt state.log index == view | throw s!"no node has a committed signature at {index} in term {view}" - replay.run header (.local joiner (.initAsBackup (source.log.take index))) step.name + let replay ← + replay.run header (.local joiner (.initAsBackup (source.log.take index))) + step.name + return { replay with snapshots := (joiner, index) :: replay.snapshots } structure Result where instructions : Nat diff --git a/lean/consensus/Consensus/Tests/ReplayDifferences.lean b/lean/consensus/Consensus/Tests/ReplayDifferences.lean new file mode 100644 index 00000000000..a03d4f47c1f --- /dev/null +++ b/lean/consensus/Consensus/Tests/ReplayDifferences.lean @@ -0,0 +1,96 @@ +module + +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. + +meta import Consensus.Replay.Interpreter + +public import Consensus.Replay.Interpreter + +public section + +namespace Consensus.Tests.ReplayDifferences + +open Consensus Consensus.Replay Consensus.Model.Node +open CCF (Finset) + +private def header : Header := { bootstrap := {0}, declared := ["0", "1", "2"] } + +private def configuration (request : ReqID) (nodes : Finset NodeID) : Entry := + { term := 2, content := .transaction request { configuration := some nodes } } + +private def signature : Entry := { term := 2, content := .signature } + +/-- Node 1 as a follower in term 2 which holds `log`. -/ +private def follower (log : List Entry) : Model.State := + let initial := header.initial + let node : NodeState := + { + Model.initialNodeState header.bootstrap 1 with + role := .follower, currentTerm := 2, log + } + { + initial with + nodes := initial.nodes.map fun (id, state) => (id, if id = 1 then node else state) + } + +/-- Node 2 retires before the signature at 3, and node 1 joins after it. -/ +private def retiredBeforeSnapshot : List Entry := + [ + configuration 0 {0, 2}, + configuration 1 {0}, + signature, + configuration 2 {0, 1}, + signature + ] + +private def heartbeat (target : NodeID) : Envelope := + { + source := 1 + target + payload := + .appendEntriesRequest + { + term := 2, + prevLogIndex := 5, + prevLogTerm := 2, + entries := [], + primaryCommit := 3 + } + } + +/-- Whether the model sent `queued` and CCF nothing, for node 1 which started at `snapshot`. -/ +private def unsent (snapshot : Nat) (queued : List Envelope) + : Except String ReplayState := + runInstruction header + { state := follower retiredBeforeSnapshot, queued, snapshots := [(1, snapshot)] } + (.observeSends [] default) + +private def check (condition : Bool) (message : String) : IO Unit := do + unless condition do + throw (IO.userError message) + +private def rejected (result : Except String ReplayState) : Bool := + match result with + | .error error => (error.splitOn "with no native send").length == 2 + | .ok _ => false + +/-- Checks what replay accepts where CCF and the model differ, and that it accepts no more. -/ +def run : IO Unit := do + -- A node which started from a snapshot does not know a node which retired before it + check (unsent 3 [heartbeat 2]).isOk + "snapshot: entries for a node which retired before it must be lost" + check (rejected (unsent 3 [heartbeat 0])) "snapshot: entries for a peer must be sent" + check (rejected (unsent 0 [heartbeat 2])) + "no snapshot: entries for a node whose retirement is not confirmed must be sent" + let response : Envelope := + { + heartbeat 2 with + payload := .requestVoteResponse { term := 2, voteGranted := true } + } + check (rejected (unsent 3 [response])) "snapshot: a response must be sent to any node" + IO.println "Replay difference checks passed." + +#eval run + +end Consensus.Tests.ReplayDifferences diff --git a/lean/consensus/README.md b/lean/consensus/README.md index ecf5fa25f1e..7feed4f4551 100644 --- a/lean/consensus/README.md +++ b/lean/consensus/README.md @@ -29,6 +29,7 @@ Like CCF's `init_as_backup`, a fresh node may initialize once as a follower: wit Crashes are not modeled, so a committed prefix keeps its history, and a valid snapshot, fetched from another node or read from a local file, is such a prefix ([`Model.networkStep`](Consensus/Model.lean)). The join protocol is not modeled: a configuration may add any node, and a fresh node may instead follow from its first AppendEntries. Node identities are never reused, so a configuration may not name a node removed by an earlier configuration in the same log. +A node which starts from a snapshot holds the log which precedes it, and so knows the nodes which retired before the snapshot until their retirement is confirmed, where CCF gives such a node only the configuration at the snapshot; [replay](replay/README.md#replay-contract) treats what the model sends to those nodes as lost. There are no fairness assumptions or guarantees of eventual election or commitment. ## Validation diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index d3171806c0b..5915c2386d0 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -86,6 +86,10 @@ A `nomination` whose snapshot is of a primary runs the model's nomination even i CCF ignores an AppendEntries response from a node which it does not know, and traces it as `recv_append_entries_response_unknown_node`. Replay delivers that response too, and the model decides by itself whether it knows the sender. +Where CCF acts on fewer messages than the model does, for a reason which the model lacks, replay treats those messages as lost, which the model allows. +A node which starts from a snapshot is given the configuration at that snapshot, so as a primary it sends neither entries nor a nomination to a node which retired before it. +The model reads the peers of a node from its whole log, and sends both until the retirement is confirmed: replay treats what it sends to such a node as lost. + ## Source and checks - [`Input.lean`](../Consensus/Replay/Input.lean): decode events, carry optional command provenance and collect identities. @@ -95,6 +99,7 @@ Replay delivers that response too, and the model decides by itself whether it kn - [`Session.lean`](../Consensus/Replay/Session.lean): replay of one driver command at a time. - [`ReplayMain.lean`](ReplayMain.lean): CLI which validates one trace, whole or by command, or reports the outcomes of several. - [`ReplayDelivery.lean`](../Consensus/Tests/ReplayDelivery.lean): delivery selection among equal rendered packets. +- [`ReplayDifferences.lean`](../Consensus/Tests/ReplayDifferences.lean): the differences between CCF and the model which replay accepts, and that it accepts no more. - [`ReplayOutcomes.lean`](../Consensus/Tests/ReplayOutcomes.lean): outcomes of chosen steps, and the report of several traces. - [`ReplaySession.lean`](../Consensus/Tests/ReplaySession.lean): replay by command against replay of the whole trace, the model state between commands, and a nomination and a response which the model decides how to handle. - [`ModelDriver.lean`](../../../tests/cft_model/ModelDriver.lean): the consensus model driver, which replays the scenarios in the process which runs them. diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index 4bb26079de5..ac67331b6ad 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -22,6 +22,7 @@ import Scenarios.follower_rollback_match_index import Scenarios.heartbeat_before_stale_suffix import Scenarios.indirect_commit_stale_leader import Scenarios.join_from_snapshot +import Scenarios.join_from_snapshot_after_retirement import Scenarios.large_entry_batching import Scenarios.late_nomination import Scenarios.leadership_race diff --git a/tests/cft_scenarios/Scenarios/join_from_snapshot_after_retirement.lean b/tests/cft_scenarios/Scenarios/join_from_snapshot_after_retirement.lean new file mode 100644 index 00000000000..25e321c483c --- /dev/null +++ b/tests/cft_scenarios/Scenarios/join_from_snapshot_after_retirement.lean @@ -0,0 +1,88 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +private def assertPending (source target : NodeId) (count : Nat) : IO Unit := do + let pending ← pendingMessages source target + let link := s!"from {source.text} to {target.text}" + unless pending.size == count do + throw (IO.userError s!"Expected {count} messages {link}, got {pending}") + +-- A node which joins from a snapshot does not know a node which retired +-- before it. Node 2 retires, and node 3 then joins from a snapshot whose +-- configuration is nodes 0 and 1. When node 3 is elected, the retirement of +-- node 2 is not confirmed, so nodes 0 and 1 still know node 2. Node 3 sends it +-- nothing, and confirms its retirement all the same. +-- Found by `cft_model_driver --explore`. +initialize scenario "join_from_snapshot_after_retirement" do + startNode 0 + trustNodes 2 [1, 2] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + -- Initial network state [0, 1, 2] + assertCommitIdx 0 3 + assertCommitIdx 1 3 + assertCommitIdx 2 3 + + -- 2 retires at 2.4, and every node commits its retirement + swapNodes 2 (retire := [2]) + replicate 2 [signature] + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 5 + assertCommitIdx 1 5 + assertDetail 2 "retirement_phase" "Completed" + + -- 3 joins from a snapshot at 2.5, which 2 has left + trustNodeFromSnapshot 2 3 + assertLastTxid 3 "2.5" + assertConfig 3 5 [0, 1] + replicate 2 [signature] + periodicAll 10 + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 7 + assertCommitIdx 3 7 + + -- 3 is elected in term 3, with the vote of 1 + periodicOne 3 100 + assertDetail 3 "leadership_state" "PreVoteCandidate" + dispatchOne 3 + dispatchOne 1 + assertDetail 3 "leadership_state" "Candidate" + dispatchOne 3 + dispatchOne 1 + assertDetail 3 "leadership_state" "Leader" + + -- 3 sends its first entries to 0 and 1, and not to 2 + assertPending 3 0 1 + assertPending 3 1 1 + assertPending 3 2 0 + dispatchAll + + -- 3 confirms the retirement of 2 at 3.8, and commits it at 3.9 + cleanupRetired 3 + assertLastTxid 3 "3.8" + replicate 3 [signature] + periodicOne 3 10 + assertPending 3 2 0 + dispatchAll + assertCommitIdx 3 9 + periodicOne 3 10 + dispatchAll + + assertDetail 0 "leadership_state" "Follower" + assertCommitIdx 0 9 + assertDetail 1 "leadership_state" "Follower" + assertCommitIdx 1 9 + assertDetail 3 "leadership_state" "Leader" + assertLastTxid 2 "2.7" From 86b70d659b6dc64798d197b83fe6fb3c603548dd Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 15:58:33 +0000 Subject: [PATCH 27/31] Replay a nomination ignored before the election timer starts CCF starts the election timer of a node when it appends or receives a signature while it is in its latest configuration, and never stops it. Until then the node ignores a nomination. The consensus model has no timer: a node campaigns if a signature covers an active configuration which names it, or if its retirement is committed and not confirmed. So a node which is added and removed between two signatures, or which retires before it first signs, ignores a nomination for which the model campaigns. Replay now treats a nomination which CCF ignores as lost if the model would campaign and the log of the node holds no signature which found it in its latest configuration. Otherwise it delivers the nomination, and the model must ignore it too. cft_model_driver --explore found the difference in 1 of 20,000 changed scenarios. The scenarios nominate_node_removed_before_signature and nominate_node_retired_before_signature show the two cases. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Consensus/Replay/Interpreter.lean | 45 ++++++++++++++++- .../consensus/Consensus/Replay/Reduction.lean | 23 ++++++--- .../Consensus/Tests/ReplayDifferences.lean | 27 +++++++++- lean/consensus/README.md | 1 + lean/consensus/replay/README.md | 3 ++ tests/cft_scenarios/Scenarios.lean | 2 + ...ominate_node_removed_before_signature.lean | 49 +++++++++++++++++++ ...ominate_node_retired_before_signature.lean | 38 ++++++++++++++ 8 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 tests/cft_scenarios/Scenarios/nominate_node_removed_before_signature.lean create mode 100644 tests/cft_scenarios/Scenarios/nominate_node_retired_before_signature.lean diff --git a/lean/consensus/Consensus/Replay/Interpreter.lean b/lean/consensus/Consensus/Replay/Interpreter.lean index 7760d038f45..4c7f8d4b561 100644 --- a/lean/consensus/Consensus/Replay/Interpreter.lean +++ b/lean/consensus/Consensus/Replay/Interpreter.lean @@ -48,6 +48,11 @@ def Header.initial (header : Header) : Model.State := inductive Step where | local (node : String) (input : Input) | receive (source destination : String) (packet : Json) + /-- + A vote proposal which CCF received and ignored. It is lost if the model would campaign + and the log of the node does not show that CCF has started its election timer. + -/ + | declineProposal (source destination : String) (packet : Json) /-- Initialize from a node's committed prefix ending at `index` in term `view`. -/ | initAsBackup (node : String) (index view : Nat) @@ -57,7 +62,7 @@ def Step.name : Step → String | .local _ (.timeout _) => "timeout" | .local _ .nominateSuccessor => "nominateSuccessor" | .local _ (.initAsBackup _) | .initAsBackup .. => "initAsBackup" - | .receive .. => "receive" + | .receive .. | .declineProposal .. => "receive" structure ReplayState where state : Model.State @@ -309,6 +314,36 @@ private def toUnknownPeer (replay : ReplayState) (envelope : Envelope) : Bool := decide (envelope.target ∈ removedNodes (state.log.take index)) | _, _, _ => false +/-- +Whether the log of `node` shows that CCF has started its election timer. CCF starts it when +the node appends or receives a signature while it is in its latest configuration, and never +stops it. A node receives no signature of the snapshot from which it started. Without such +a signature, the timer may still have started, since a rollback may have removed the +signature which started it. +-/ +private def electionTimerStarted (replay : ReplayState) (node : NodeID) (log : List Entry) + : Bool := + let snapshot := (replay.snapshots.lookup node).getD 0 + (List.range (log.length + 1)).any + fun index => + snapshot < index + && isSignatureAt log index + && decide + (node ∈ latestConfigurationMembers (configurationHistory (log.take index))) + +/-- +Whether CCF may ignore a vote proposal for which the model campaigns: it does so until it +has started the election timer of the node, which the model does not have. +-/ +private def mayDeclineProposal (replay : ReplayState) (destination : NodeID) + (envelope : Envelope) + : Bool := + match envelope.payload, nodeState replay.state destination with + | .proposeRequestVoteRequest term, some state => + decide (term = state.currentTerm /\ canStartElection state destination) + && !electionTimerStarted replay destination state.log + | _, _ => false + /-- Observations never change model state or repair a disabled action. -/ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instruction) : Except String ReplayState := do @@ -355,6 +390,14 @@ def runInstruction (header : Header) (replay : ReplayState) (instruction : Instr let destination ← header.nodeId destination let envelope ← selectPacket replay source destination packet replay.run header (.deliver envelope) "receive" + | .action (.declineProposal source destination packet) _ => + let source ← header.nodeId source + let destination ← header.nodeId destination + let envelope ← selectPacket replay source destination packet + if mayDeclineProposal replay destination envelope then + -- Loss is non-delivery + return replay + replay.run header (.deliver envelope) "receive" | .action (step@(.initAsBackup node index view)) _ => let joiner ← header.nodeId node -- Committed prefixes in a reachable state are comparable, so any such node will do. diff --git a/lean/consensus/Consensus/Replay/Reduction.lean b/lean/consensus/Consensus/Replay/Reduction.lean index 34b06fe557b..f9792c824d9 100644 --- a/lean/consensus/Consensus/Replay/Reduction.lean +++ b/lean/consensus/Consensus/Replay/Reduction.lean @@ -272,15 +272,22 @@ private def ruleRecvProposeRequestVote : Reduction Unit := do -- consume let start ← position event head "recv_propose_request_vote" - optional_ do - event head "become_candidate" - optional_ do - event head "become_leader" + let campaigned ← + CCF.Parsec.optionalRule do + event head "become_candidate" optional_ do - event head "commit" - optional_ (event head "step_down_and_nominate_successor") - many_ (event head "send_append_entries") - many_ (event head "send_request_vote") + event head "become_leader" + optional_ do + event head "commit" + optional_ (event head "step_down_and_nominate_successor") + many_ (event head "send_append_entries") + many_ (event head "send_request_vote") + -- CCF ignores a proposal until it has started the election timer of the node + let action := + match action, campaigned with + | .receive source destination packet, none => + .declineProposal source destination packet + | action, _ => action -- emit emitObserve head "recv-propose-request-vote" emitAction head "recv-propose-request-vote" action diff --git a/lean/consensus/Consensus/Tests/ReplayDifferences.lean b/lean/consensus/Consensus/Tests/ReplayDifferences.lean index a03d4f47c1f..57adb5ed34a 100644 --- a/lean/consensus/Consensus/Tests/ReplayDifferences.lean +++ b/lean/consensus/Consensus/Tests/ReplayDifferences.lean @@ -13,6 +13,7 @@ namespace Consensus.Tests.ReplayDifferences open Consensus Consensus.Replay Consensus.Model.Node open CCF (Finset) +open CCF.MultiNodeTransitionSystem (nodeState) private def header : Header := { bootstrap := {0}, declared := ["0", "1", "2"] } @@ -21,8 +22,8 @@ private def configuration (request : ReqID) (nodes : Finset NodeID) : Entry := private def signature : Entry := { term := 2, content := .signature } -/-- Node 1 as a follower in term 2 which holds `log`. -/ -private def follower (log : List Entry) : Model.State := +/-- Node 1 as a follower in term 2 which holds `log`, with `network` to deliver. -/ +private def follower (log : List Entry) (network : List Envelope := []) : Model.State := let initial := header.initial let node : NodeState := { @@ -32,6 +33,7 @@ private def follower (log : List Entry) : Model.State := { initial with nodes := initial.nodes.map fun (id, state) => (id, if id = 1 then node else state) + network } /-- Node 2 retires before the signature at 3, and node 1 joins after it. -/ @@ -66,6 +68,17 @@ private def unsent (snapshot : Nat) (queued : List Envelope) { state := follower retiredBeforeSnapshot, queued, snapshots := [(1, snapshot)] } (.observeSends [] default) +private def proposal : Envelope := + { source := 0, target := 1, payload := .proposeRequestVoteRequest 2 } + +/-- Node 1 ignores a proposal of node 0, and sends nothing. -/ +private def ignored (log : List Entry) : Except String ReplayState := do + let packet := messageJson proposal.payload + let replay ← + runInstruction header { state := follower log [proposal] } + (.action (.declineProposal "0" "1" packet) default) + runInstruction header replay (.observeSends [] default) + private def check (condition : Bool) (message : String) : IO Unit := do unless condition do throw (IO.userError message) @@ -89,6 +102,16 @@ def run : IO Unit := do payload := .requestVoteResponse { term := 2, voteGranted := true } } check (rejected (unsent 3 [response])) "snapshot: a response must be sent to any node" + -- A node ignores a proposal until a signature finds it in its latest configuration + match ignored [configuration 0 {0, 1}, configuration 1 {0}, signature] with + | .ok replay => + check + (replay.state.network == [proposal] + && (nodeState replay.state 1).map (·.role) == some .follower) + "no timer: the proposal must be lost" + | .error error => throw (IO.userError s!"no timer: {error}") + check (rejected (ignored [configuration 0 {0, 1}, signature])) + "timer: the model must campaign, and CCF with it" IO.println "Replay difference checks passed." #eval run diff --git a/lean/consensus/README.md b/lean/consensus/README.md index 7feed4f4551..71f21cb8edd 100644 --- a/lean/consensus/README.md +++ b/lean/consensus/README.md @@ -30,6 +30,7 @@ Crashes are not modeled, so a committed prefix keeps its history, and a valid sn The join protocol is not modeled: a configuration may add any node, and a fresh node may instead follow from its first AppendEntries. Node identities are never reused, so a configuration may not name a node removed by an earlier configuration in the same log. A node which starts from a snapshot holds the log which precedes it, and so knows the nodes which retired before the snapshot until their retirement is confirmed, where CCF gives such a node only the configuration at the snapshot; [replay](replay/README.md#replay-contract) treats what the model sends to those nodes as lost. +The election timer is not modeled: a node may campaign once a signature covers an active configuration which names it, or while its retirement is committed and not confirmed, where CCF waits for the node to hold a signature while it is in its latest configuration; replay treats a nomination which CCF ignores for that reason as lost. There are no fairness assumptions or guarantees of eventual election or commitment. ## Validation diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index 5915c2386d0..c768ba5937d 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -89,6 +89,9 @@ Replay delivers that response too, and the model decides by itself whether it kn Where CCF acts on fewer messages than the model does, for a reason which the model lacks, replay treats those messages as lost, which the model allows. A node which starts from a snapshot is given the configuration at that snapshot, so as a primary it sends neither entries nor a nomination to a node which retired before it. The model reads the peers of a node from its whole log, and sends both until the retirement is confirmed: replay treats what it sends to such a node as lost. +CCF ignores a `propose_request_vote` until the election timer of the node has started, which is when it appends or receives a signature while it is in its latest configuration. +The model has no timer: it campaigns if a signature covers an active configuration which names the node, or if the retirement of the node is committed and not confirmed. +Replay treats a proposal which CCF ignores as lost if the model would campaign and the log of the node holds no such signature, and delivers it otherwise, so that the model must ignore it too. ## Source and checks diff --git a/tests/cft_scenarios/Scenarios.lean b/tests/cft_scenarios/Scenarios.lean index ac67331b6ad..d640ea476fe 100644 --- a/tests/cft_scenarios/Scenarios.lean +++ b/tests/cft_scenarios/Scenarios.lean @@ -31,6 +31,8 @@ import Scenarios.matching_partial import Scenarios.message_type_summaries import Scenarios.multi_election import Scenarios.nominate_new_node +import Scenarios.nominate_node_removed_before_signature +import Scenarios.nominate_node_retired_before_signature import Scenarios.nominate_over_retiring_node import Scenarios.nominate_successor import Scenarios.nominate_without_successor diff --git a/tests/cft_scenarios/Scenarios/nominate_node_removed_before_signature.lean b/tests/cft_scenarios/Scenarios/nominate_node_removed_before_signature.lean new file mode 100644 index 00000000000..9db18f4bd1e --- /dev/null +++ b/tests/cft_scenarios/Scenarios/nominate_node_removed_before_signature.lean @@ -0,0 +1,49 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node ignores a nomination until its election timer starts, which is when +-- it holds a signature while it is in its latest configuration. Node 1 is +-- added and removed between two signatures, so no signature finds it in its +-- latest configuration, though the second one covers the configuration which +-- added it, and that configuration is still active. +-- Found by `cft_model_driver --explore`. +initialize scenario "nominate_node_removed_before_signature" do + startNode 0 + replicate 2 [signature] + + -- 1 and 2 join at 2.3, 1 retires at 2.4, and 0 signs at 2.5 + swapNodes 2 [] [1, 2] + swapNodes 2 (retire := [1]) + replicate 2 [signature] + + -- Only 1 receives the entries, so 0 commits none of them + disconnect 0 2 + periodicOne 0 10 + dispatchAll + periodicOne 0 10 + dispatchAll + assertLastTxid 1 "2.5" + assertDetail 1 "retirement_phase" "Signed" + assertCommitIdx 0 2 + assertConfig 0 3 [0, 1, 2] + + -- 0 nominates 1, which has the most entries, and 1 ignores it + nominateSuccessor 0 + dispatchSingle 0 1 + assertDetail 1 "leadership_state" "Follower" + assertDetail 1 "current_view" 2 + + -- 2 catches up, and 0 commits the retirement of 1 + reconnect 0 2 + periodicOne 0 10 + dispatchAll + periodicOne 0 10 + dispatchAll + + assertDetail 0 "leadership_state" "Leader" + assertCommitIdx 0 5 + assertAbsentConfig 0 3 + assertLastTxid 2 "2.5" diff --git a/tests/cft_scenarios/Scenarios/nominate_node_retired_before_signature.lean b/tests/cft_scenarios/Scenarios/nominate_node_retired_before_signature.lean new file mode 100644 index 00000000000..db94e9df631 --- /dev/null +++ b/tests/cft_scenarios/Scenarios/nominate_node_retired_before_signature.lean @@ -0,0 +1,38 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the Apache 2.0 License. +import CftDriver + +open CftDriver + +-- A node which is never in its latest configuration when it holds a signature +-- does not start its election timer, and ignores a nomination. Node 0 replaces +-- itself with node 1 before it first signs. Its retirement is committed, and +-- not confirmed, when node 1 is elected and nominates it. +-- Found by `cft_model_driver --explore`. +initialize scenario "nominate_node_retired_before_signature" do + startNode 0 + + -- 0 retires and 1 joins at 2.2, before 0 signs at 2.3 + swapNodes 2 [0] [1] + replicate 2 [signature] + dispatchAll + periodicAll 10 + dispatchAll + assertCommitIdx 0 3 + assertDetail 0 "retirement_phase" "Completed" + + -- 0 stands down for want of an acknowledgement, and 1 is elected in term 3 + periodicOne 0 100 + assertDetail 0 "leadership_state" "Follower" + dispatchAll + assertCommitIdx 1 3 + periodicOne 1 100 + assertDetail 1 "leadership_state" "Leader" + assertDetail 1 "current_view" 3 + + -- 1 nominates 0, the only node which it knows, and 0 ignores it + nominateSuccessor 1 + dispatchAll + assertDetail 0 "leadership_state" "Follower" + assertDetail 0 "current_view" 3 + assertDetail 1 "leadership_state" "Leader" From 6616a628755d0e50ae8ed9ed5d9db41601f93401 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 17:28:58 +0000 Subject: [PATCH 28/31] Fail the exploration of consensus scenarios on a finding cft_model_driver --explore changes the consensus scenarios at random, and reports as a finding a run in which the driver fails or the model rejects the trace. It exited with status 0 whatever it found, and cft_model_test explored 40 changed scenarios only to check that exploring works. The exploration now exits with status 1 if a run is a finding, and writes the program of each finding to its error output, so that the finding can be run again from the log of a test. cft_model_test fails on a finding as a result. With a rollback which no longer forgets the nodes of the configurations which it removes, 2,000 runs with the default seed include 9 of one finding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/cft_model/ModelDriver.lean | 5 +++-- tests/cft_model/ModelDriver/Explore.lean | 9 +++++++-- tests/cft_model/README.md | 4 +++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/cft_model/ModelDriver.lean b/tests/cft_model/ModelDriver.lean index 9dc351abe93..21c0f6b86b8 100644 --- a/tests/cft_model/ModelDriver.lean +++ b/tests/cft_model/ModelDriver.lean @@ -158,7 +158,8 @@ private def roundTrip (directory : System.FilePath) (name : String) : IO String /-- Run every scenario and the run in step, each in a process of its own, because a process -can start the driver once. Also run each scenario from its commands, and explore briefly. +can start the driver once. Also run each scenario from its commands, and explore briefly, +which fails if a run is a finding. -/ private def runAll : IO UInt32 := do let names := (← scenarios.get).map (·.1) |>.reverse @@ -174,7 +175,7 @@ private def runAll : IO UInt32 := do else failed := failed + 1 IO.println s!"{name}: FAILED\n{problem}" - -- A short exploration must run, whatever it finds + -- A short exploration, in which no run may be a finding let (status, errors) ← again #["--explore", (directory / "explore").toString, "--runs", "40"] if status == 0 then diff --git a/tests/cft_model/ModelDriver/Explore.lean b/tests/cft_model/ModelDriver/Explore.lean index e6eb846af59..452060973be 100644 --- a/tests/cft_model/ModelDriver/Explore.lean +++ b/tests/cft_model/ModelDriver/Explore.lean @@ -599,7 +599,8 @@ def shrink (file output : System.FilePath) (keep : Keep) (jobs : Nat) : IO UInt3 /-- Record the commands of each scenario, change them at random and run the changed programs. -Write the programs kept and found, and a report, to the directory. +Write the programs kept and found, and a report, to the directory. The status is 1 if a +run is a finding. -/ def explore (options : Exploration) (scenarios : List String) : IO UInt32 := do let directory := options.directory @@ -671,6 +672,10 @@ def explore (options : Exploration) (scenarios : List String) : IO UInt32 := do let report := summary options state seconds IO.FS.writeFile (directory / "report.md") report IO.println report - return 0 + -- A finding fails the exploration, and its program is what shows it again + for index in [0:state.findings.size] do + let file := directory / "findings" / s!"{index}.program" + IO.eprintln s!"finding {file}:\n{← IO.FS.readFile file}" + return if state.findings.isEmpty then 0 else 1 end ModelDriver diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index c064707ceff..ed38d7c2fcc 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -16,7 +16,7 @@ build/cft_model_driver --all It replays each command as soon as the driver has run it, and requires each undelivered message of the driver to be a message which the model holds. It delivers the messages which the model holds, newest first, and asserts of the driver the commit indexes of the model. - `--all` runs every scenario and the run in step, each in a process of its own, because a process can start the driver once. - It also runs each scenario again from its [recorded commands](#programs), which the model must accept and which must all apply, and [explores](#exploring) briefly. + It also runs each scenario again from its [recorded commands](#programs), which the model must accept and which must all apply, and [explores](#exploring) briefly, which must give no finding. The two runs of a scenario may differ where an election timeout decides, because the timeout of a node differs between processes. A run prints its status as JSON. @@ -61,6 +61,8 @@ It runs each changed program in a process of its own. The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, the edges which the scenarios reach in `scenarios.edges`, and `report.md`. The same seed gives the same changes. +The exploration exits with status 1 if a run is a finding, and writes the program of each finding to its error output. +The election timeout of a node differs between processes, so a finding which depends on one may not show in every exploration with the same seed. ```sh build/cft_model_driver --shrink [--outcome | --edges ] From acc35f9aad8f151f02ee9f4796074c0fee35ed30 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 17:29:21 +0000 Subject: [PATCH 29/31] Explore changes to the consensus scenarios in the Lean workflow The capture job of the Lean workflow builds cft_model_driver and runs every scenario with the consensus model. It now also explores 2,000 changed scenarios with a fixed seed, and fails if the implementation and the model disagree in one of them. It uploads the findings when it fails. The same seed gives the same changes, so the step tells whether a change breaks what those runs reach, and looks for nothing new. In a Debug build like that of the job, the exploration takes 87 seconds with 8 runs at once. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/lean.yml | 22 ++++++++++++++++++++++ doc/architecture/verification.rst | 2 +- tests/cft_model/README.md | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index a1c459e7b0b..bad8b151b00 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -92,6 +92,28 @@ jobs: cmake --build "$RUNNER_TEMP/cft-build" --target cft_model_test --parallel "$(( ($(nproc) + 1) >> 1 ))" ctest --test-dir "$RUNNER_TEMP/cft-build" -VV -R "^cft_model_test$" --no-tests=error + # Fails if the implementation and the model disagree in a changed scenario. + - name: Explore changes to the scenarios + timeout-minutes: 15 + run: | + set -euo pipefail + # RUNNER_TEMP can persist between jobs on reused self-hosted runners; drop files from earlier runs. + rm -rf "$RUNNER_TEMP/cft-explore" + "$RUNNER_TEMP/cft-build/cft_model_driver" --explore "$RUNNER_TEMP/cft-explore" \ + --seed 1 --runs 2000 --jobs "$(( ($(nproc) + 1) >> 1 ))" + + - name: Upload exploration findings + if: ${{ failure() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: cft-exploration + path: | + ${{ runner.temp }}/cft-explore/report.md + ${{ runner.temp }}/cft-explore/findings + if-no-files-found: ignore + retention-days: 7 + overwrite: true + kv_traces: name: KV trace captures runs-on: [gha-vmss-d16av7-ci-c] diff --git a/doc/architecture/verification.rst b/doc/architecture/verification.rst index 2453029aa33..da2f6c6abe0 100644 --- a/doc/architecture/verification.rst +++ b/doc/architecture/verification.rst @@ -82,7 +82,7 @@ The :ccf_repo:`Lean workflow <.github/workflows/lean.yml>` runs on pull requests * A package job for each of ``common``, ``consensus``, ``consistency``, ``disaster-recovery`` and ``kv`` runs the kernel and axiom check (``lake check``), import coverage and architecture checks, formatting and linting, and, where applicable, the package's replay fixtures. * Separate capture jobs build CCF and record consensus, KV and consistency traces from the tests described above. - The consensus capture job also runs ``cft_model_test``. + The consensus capture job also runs ``cft_model_test``, and then ``cft_model_driver --explore`` with a fixed seed, which fails if the implementation and the model disagree in a changed scenario. * Replay jobs build each replayer and validate every captured trace. The consensus replay job also reports which outcomes of the model's handlers the captured traces show, and which none shows; see :ccf_repo:`lean/consensus/replay/README.md`. * A final ``Lean models and proofs`` job succeeds only if all of the above succeed. diff --git a/tests/cft_model/README.md b/tests/cft_model/README.md index ed38d7c2fcc..435fd1ffc7e 100644 --- a/tests/cft_model/README.md +++ b/tests/cft_model/README.md @@ -62,6 +62,7 @@ It runs each changed program in a process of its own. The directory receives the programs kept in `corpus/`, the first program and trace of each finding in `findings/`, the edges which the scenarios reach in `scenarios.edges`, and `report.md`. The same seed gives the same changes. The exploration exits with status 1 if a run is a finding, and writes the program of each finding to its error output. +The [Lean workflow](../../.github/workflows/lean.yml) explores in this way with a fixed seed, so that a change which makes the implementation and the model disagree in one of those runs fails CI. The election timeout of a node differs between processes, so a finding which depends on one may not show in every exploration with the same seed. ```sh From 43715baa9b1987114d0cf9722bf444b11c813fc0 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 17:29:22 +0000 Subject: [PATCH 30/31] Fail the Lean workflow when scenarios stop showing an outcome The replay job reports which outcomes of the model the traces of the scenarios show. A scenario still passes when a change makes it stop showing the outcome for which it was written, and 11 outcomes are shown by one scenario only. consensus-replay --outcomes takes --not-shown and the names of outcomes, and fails unless the outcomes which no trace shows are those named. The replay job names the one outcome which no scenario shows, so that losing another fails it, and so does showing that one without removing its name. 30 captures of the 82 scenarios in each of a Debug and a RelWithDebInfo build all show the same 78 outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/lean.yml | 2 ++ doc/architecture/verification.rst | 2 +- lean/consensus/Consensus/Replay/Outcomes.lean | 6 +++- .../Consensus/Tests/ReplayOutcomes.lean | 3 ++ lean/consensus/replay/README.md | 8 +++++ lean/consensus/replay/ReplayMain.lean | 33 ++++++++++++++++--- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index bad8b151b00..1634be015a8 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -303,12 +303,14 @@ jobs: echo "::endgroup::" done + # An outcome which the scenarios showed and no longer show fails this step. - name: Report model outcomes shown by CCF consensus traces shell: bash run: | set -euo pipefail shopt -s failglob lean/consensus/.lake/build/bin/consensus-replay --outcomes \ + --not-shown appendEntriesConflictIgnored \ "$RUNNER_TEMP"/cft-scenario-traces/*.ndjson >> "$GITHUB_STEP_SUMMARY" models: diff --git a/doc/architecture/verification.rst b/doc/architecture/verification.rst index da2f6c6abe0..f3c1e84dec5 100644 --- a/doc/architecture/verification.rst +++ b/doc/architecture/verification.rst @@ -84,7 +84,7 @@ The :ccf_repo:`Lean workflow <.github/workflows/lean.yml>` runs on pull requests * Separate capture jobs build CCF and record consensus, KV and consistency traces from the tests described above. The consensus capture job also runs ``cft_model_test``, and then ``cft_model_driver --explore`` with a fixed seed, which fails if the implementation and the model disagree in a changed scenario. * Replay jobs build each replayer and validate every captured trace. - The consensus replay job also reports which outcomes of the model's handlers the captured traces show, and which none shows; see :ccf_repo:`lean/consensus/replay/README.md`. + The consensus replay job also reports which outcomes of the model's handlers the captured traces show, and which none shows, and fails if the traces no longer show an outcome which they did; see :ccf_repo:`lean/consensus/replay/README.md`. * A final ``Lean models and proofs`` job succeeds only if all of the above succeed. The disaster recovery traces are replayed by the SNP Genoa job in the main CI workflow instead, since that is where the recovery scenarios run. diff --git a/lean/consensus/Consensus/Replay/Outcomes.lean b/lean/consensus/Consensus/Replay/Outcomes.lean index 16c4dc42a4e..489abb945eb 100644 --- a/lean/consensus/Consensus/Replay/Outcomes.lean +++ b/lean/consensus/Consensus/Replay/Outcomes.lean @@ -568,6 +568,10 @@ def outcomes (state : Model.State) (action : Model.Action) : List Outcome := | .receive source message => receiveOutcomes s source message handled ++ electedOutcomes s ++ commitOutcomes s ++ retirementOutcomes s +/-- The outcomes which none of the traces shows, in the order of `Outcome.all`. -/ +def notShown (traces : Array (String × Array Outcome)) : List Outcome := + Outcome.all.filter fun outcome => traces.all fun (_, shown) => !shown.contains outcome + /-- A Markdown table of the steps and traces which show each outcome, with the first such trace, followed by the outcomes which no trace shows. @@ -579,7 +583,7 @@ def report (traces : Array (String × Array Outcome)) : String := let showing := traces.filter fun (_, shown) => shown.contains outcome let steps := traces.foldl (fun total (_, shown) => total + shown.count outcome) 0 (outcome.name, steps, showing.size, (showing[0]?.map Prod.fst).getD "") - let missing := (rows.filter fun (_, steps, _) => steps == 0).map (·.1) + let missing := (notShown traces).map (·.name) let lines := ["| Outcome | Steps | Traces | First trace |", "| --- | ---: | ---: | --- |"] ++ rows.map diff --git a/lean/consensus/Consensus/Tests/ReplayOutcomes.lean b/lean/consensus/Consensus/Tests/ReplayOutcomes.lean index d8651abdd7c..b23e1474f61 100644 --- a/lean/consensus/Consensus/Tests/ReplayOutcomes.lean +++ b/lean/consensus/Consensus/Tests/ReplayOutcomes.lean @@ -132,6 +132,9 @@ private def reporting : IO Unit := do for line in expected do unless lines.contains line do throw (IO.userError s!"report: missing line '{line}'") + let missing := notShown traces + unless missing.length + 1 == Outcome.all.length && !missing.contains .voteGranted do + throw (IO.userError "not shown: expected every outcome but the one which is shown") /-- Checks the outcomes of chosen steps, and the report of several traces. -/ def run : IO Unit := do diff --git a/lean/consensus/replay/README.md b/lean/consensus/replay/README.md index c768ba5937d..22ca5e1f910 100644 --- a/lean/consensus/replay/README.md +++ b/lean/consensus/replay/README.md @@ -47,6 +47,14 @@ An outcome which no trace shows is a missing scenario, a step which the driver o For example, no trace of a scenario shows `appendEntriesConflictIgnored`, which [`cft.h`](../../../src/consensus/aft/cft.h) logs as a branch which it expects not to reach. The [consensus model driver](../../../tests/cft_model/README.md#exploring) looks for runs which show such an outcome, by changing the scenarios at random. +With `--not-shown` and the names of outcomes, separated by commas, the report also fails unless the outcomes which no trace shows are those named: + +```sh +./.lake/build/bin/consensus-replay --outcomes --not-shown appendEntriesConflictIgnored /tmp/cft-build/consensus/raw/*.ndjson +``` + +The [Lean workflow](../../../.github/workflows/lean.yml) does so, so that a change after which the scenarios no longer show an outcome fails CI, and the name of an outcome which a new scenario shows is removed there. + To compare with the implementation, configure the scenario build with `-DCOVERAGE=ON` and run the scenario test. [`coverage.sh`](../../../scripts/coverage.sh) then lists the lines of [`cft.h`](../../../src/consensus/aft/cft.h) which the scenarios do not execute: diff --git a/lean/consensus/replay/ReplayMain.lean b/lean/consensus/replay/ReplayMain.lean index e2cb0f75b20..9f1cd508509 100644 --- a/lean/consensus/replay/ReplayMain.lean +++ b/lean/consensus/replay/ReplayMain.lean @@ -48,8 +48,12 @@ private def validateByCommand (path : String) : IO UInt32 := do | .error error, .ok _ => disagree path s!"only replay of the whole trace rejects it: {error}" -/-- Validate each trace, then report the outcomes which they show together. -/ -private def reportOutcomes (paths : List String) : IO UInt32 := do +/-- +Validate each trace, then report the outcomes which they show together. With `expected`, +the outcomes which no trace shows must be those which it names. +-/ +private def reportOutcomes (expected : Option (List String)) (paths : List String) + : IO UInt32 := do let mut traces := #[] for path in paths do match ← validate path with @@ -58,19 +62,38 @@ private def reportOutcomes (paths : List String) : IO UInt32 := do let name := (System.FilePath.mk path).fileStem.getD path traces := traces.push (name, result.outcomes) IO.println (report traces) - return 0 + let some expected := expected | return 0 + let missing := (notShown traces).map (·.name) + let known := Outcome.all.map (·.name) + let problems := + [ + ("no trace shows any longer", missing.filter (!expected.contains ·)), + ( + "a trace now shows, and --not-shown still names", + expected.filter fun name => known.contains name && !missing.contains name + ), + ("not an outcome", expected.filter (!known.contains ·)) + ] + for (problem, names) in problems do + unless names.isEmpty do + IO.eprintln s!"{problem}: {", ".intercalate names}" + return if problems.all (·.2.isEmpty) then 0 else 1 private def usage : IO UInt32 := do IO.eprintln "usage: consensus-replay TRACE.ndjson" IO.eprintln " consensus-replay --by-command TRACE.ndjson" - IO.eprintln " consensus-replay --outcomes TRACE.ndjson..." + IO.eprintln + " consensus-replay --outcomes [--not-shown OUTCOME,...] TRACE.ndjson..." return 2 def main (args : List String) : IO UInt32 := do try match args with | ["--outcomes"] | ["--by-command"] => usage - | "--outcomes" :: paths => reportOutcomes paths + | "--outcomes" :: "--not-shown" :: names :: path :: paths => + reportOutcomes (some ((names.splitOn ",").filter (!·.isEmpty))) (path :: paths) + | "--outcomes" :: "--not-shown" :: _ => usage + | "--outcomes" :: paths => reportOutcomes none paths | ["--by-command", path] => validateByCommand path | [path] => match ← validate path with From 7c38c704df2ea0c3448a4fdbeaf5dab7b9866732 Mon Sep 17 00:00:00 2001 From: achamayou Date: Sun, 15 Nov 2026 17:51:55 +0000 Subject: [PATCH 31/31] Explore 5,000 changed consensus scenarios in the Lean workflow On a runner, the 2,000 changed scenarios which the capture job explored take 33 seconds, in a job of 4.2 minutes. The longest job of the workflow takes 5.5 minutes. The job now explores 5,000. The seed is fixed, so these are the same runs each time, and more of them check more cases on every pull request which changes consensus. In a Debug build on a development machine they take 184 seconds with 8 runs at once, and none is a finding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/lean.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lean.yml b/.github/workflows/lean.yml index 1634be015a8..2576d5c6aa7 100644 --- a/.github/workflows/lean.yml +++ b/.github/workflows/lean.yml @@ -100,7 +100,7 @@ jobs: # RUNNER_TEMP can persist between jobs on reused self-hosted runners; drop files from earlier runs. rm -rf "$RUNNER_TEMP/cft-explore" "$RUNNER_TEMP/cft-build/cft_model_driver" --explore "$RUNNER_TEMP/cft-explore" \ - --seed 1 --runs 2000 --jobs "$(( ($(nproc) + 1) >> 1 ))" + --seed 1 --runs 5000 --jobs "$(( ($(nproc) + 1) >> 1 ))" - name: Upload exploration findings if: ${{ failure() }}