mod actor; mod ingress; mod round; mod slot; mod state; use crate::{ CertifiableAutomaton, Relay, Reporter, simplex::{Floor, Plan, elector::Elector, types::Activity}, types::{Epoch, ViewDelta}, }; pub use actor::Actor; use commonware_cryptography::{Digest, certificate::Scheme}; use commonware_p2p::Blocker; use commonware_runtime::buffer::paged::CacheRef; use core::num::NonZeroUsize; pub use ingress::Mailbox; #[cfg(test)] pub use ingress::Message; use std::time::Duration; pub struct Config< S: Scheme, L: Elector, B: Blocker, D: Digest, A: CertifiableAutomaton, R: Relay>, F: Reporter>, > { pub scheme: S, pub elector: L, pub blocker: B, pub automaton: A, pub relay: R, pub reporter: F, pub partition: String, pub epoch: Epoch, pub floor: Floor, pub mailbox_size: NonZeroUsize, pub leader_timeout: Duration, pub certification_timeout: Duration, pub timeout_retry: Duration, pub skip_budget: u64, pub view_retention: ViewDelta, pub replay_buffer: NonZeroUsize, pub write_buffer: NonZeroUsize, pub page_cache: CacheRef, } #[cfg(test)] mod tests { use super::*; use crate::{ Epochable, Handoff, Viewable, simplex::{ actors::{ Kind, batcher, resolver::{self, MailboxMessage}, }, elector::{ self, Config as _, Mode as _, Random, RandomVersion, RoundRobin, RoundRobinElector, }, metrics::TimeoutReason, mocks::{self, application::Certifier}, quorum, scheme::{ Scheme, bls12381_multisig, bls12381_threshold::vrf as bls12381_threshold_vrf, ed25519, secp256r1, }, types::{ Artifact, Certificate, Context, Finalization, Finalize, Notarization, Notarize, Nullification, Nullify, Proposal, Vote, }, }, types::{Participant, Round, TermLength, View}, }; use bytes::Bytes; use commonware_actor::mailbox; use commonware_codec::{Decode, DecodeExt, Encode}; use commonware_cryptography::{ Hasher as _, Sha256, bls12381::primitives::variant::{MinPk, MinSig}, certificate::{Scheme as _, Verifier as _, mocks::Fixture}, ed25519::PublicKey, sha256::Digest as Sha256Digest, }; use commonware_macros::{select, test_collect_traces, test_traced}; use commonware_p2p::{ Receiver as _, Recipients, simulated::{Config as NConfig, Link, Network, Oracle}, }; use commonware_parallel::Sequential; use commonware_runtime::{ Clock, Error, Handle, Metrics, Quota, ReadOptions, Runner, Spawner, Storage, Supervisor as _, deterministic, mocks::{DelayedSyncContext, PendingSyncs, next_pending_sync}, reschedule, telemetry::traces::collector::{RecordedEvents, TraceStorage}, }; use commonware_storage::journal::segmented::variable::{Config as JConfig, Journal}; use commonware_utils::{ NZU16, NZU32, NZUsize, channel::{mpsc, oneshot}, non_empty, probability, sync::Mutex, }; use futures::FutureExt; use rand_core::CryptoRng; use std::{ collections::BTreeMap, num::{NonZeroU16, NonZeroU32}, sync::Arc, time::Duration, }; use tracing::Level; const PAGE_SIZE: NonZeroU16 = NZU16!(1024); const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10); const TEST_QUOTA: Quota = Quota::per_second(NonZeroU32::MAX); type ProposeRequests = Arc>>>; type HandoffResponses = Arc< Mutex< Vec<( Handoff, oneshot::Sender>, )>, >, >; type CertificationRequests = Arc)>>>; fn labeled_metric_snapshot( metrics: &str, actor_prefix: &str, family: &str, label: &str, ) -> BTreeMap { let series = format!("{actor_prefix}_{family}_total"); let labeled_series = format!("{series}{{"); let value_prefix = format!("{{{label}=\""); metrics .lines() .filter_map(|line| { let (sample, count) = line.split_once(' ')?; if sample != series && !sample.starts_with(&labeled_series) { return None; } let labels = sample.strip_prefix(&series).unwrap(); let value = labels .strip_prefix(&value_prefix) .and_then(|value| value.strip_suffix("\"}")) .unwrap_or_else(|| { panic!("unexpected labels on scoped metric series: {sample}") }); Some((value.to_string(), count.parse().unwrap())) }) .fold(BTreeMap::new(), |mut snapshot, (value, count)| { assert!( snapshot.insert(value.clone(), count).is_none(), "duplicate scoped metric series for {label}={value}" ); snapshot }) } fn assert_handoff_metrics( metrics: &str, actor_prefix: &str, events: &[(&str, u64)], abandoned: &[(&str, u64)], ) { let expected_events = events .iter() .map(|(label, count)| ((*label).to_string(), *count)) .collect(); let expected_abandoned = abandoned .iter() .map(|(label, count)| ((*label).to_string(), *count)) .collect(); assert_eq!( labeled_metric_snapshot(metrics, actor_prefix, "handoff_events", "event"), expected_events ); assert_eq!( labeled_metric_snapshot(metrics, actor_prefix, "handoff_abandoned", "reason"), expected_abandoned ); } async fn start_test_network_with_peers( context: deterministic::Context, peers: I, disconnect_on_block: bool, ) -> Oracle where I: IntoIterator, { let peers: Vec<_> = peers.into_iter().collect(); let (network, oracle) = Network::new_with_peers( context.child("network"), NConfig { max_size: 1024 * 1024, max_peers_per_set: NZUsize!(peers.len()), disconnect_on_block, tracked_peer_sets: NZUsize!(1), }, peers, ) .await; network.start(); oracle } fn build_notarization>( schemes: &[S], proposal: &Proposal, count: u32, ) -> ( Vec>, Notarization, ) { let votes: Vec<_> = schemes .iter() .take(count as usize) .map(|scheme| Notarize::sign(scheme, proposal.clone()).unwrap()) .collect(); let certificate = Notarization::from_notarizes(&schemes[0], non_empty![@&votes], &Sequential) .expect("notarization requires a quorum of votes"); (votes, certificate) } fn build_finalization>( schemes: &[S], proposal: &Proposal, count: u32, ) -> ( Vec>, Finalization, ) { let votes: Vec<_> = schemes .iter() .take(count as usize) .map(|scheme| Finalize::sign(scheme, proposal.clone()).unwrap()) .collect(); let certificate = Finalization::from_finalizes(&schemes[0], non_empty![@&votes], &Sequential) .expect("finalization requires a quorum of votes"); (votes, certificate) } fn build_nullification>( schemes: &[S], round: Round, count: u32, ) -> (Vec>, Nullification) { let votes: Vec<_> = schemes .iter() .take(count as usize) .map(|scheme| Nullify::sign::(scheme, round).unwrap()) .collect(); let certificate = Nullification::from_nullifies(&schemes[0], non_empty![@&votes], &Sequential) .expect("nullification requires a quorum of votes"); (votes, certificate) } /// Returns the schemes of every participant except `local_index`. A certificate built from /// their votes lacks the local voter's share, so an honest local voter could observe it /// only from its peers. fn peers(schemes: &[S], local_index: usize) -> Vec { schemes .iter() .enumerate() .filter(|(index, _)| *index != local_index) .map(|(_, scheme)| scheme.clone()) .collect() } /// Voter knobs for [`setup_voter`], named so call sites cannot transpose /// the timeouts. Tests override only the fields they exercise. struct VoterOptions { leader_timeout: Duration, certification_timeout: Duration, timeout_retry: Duration, /// Index of the participant hosting the voter under test. local_index: usize, /// Mock application propose latency, in milliseconds. propose_latency_ms: f64, /// Mock application verify latency, in milliseconds. verify_latency_ms: f64, /// Mock application certify latency, in milliseconds. certify_latency_ms: f64, /// Contexts supplied to mock application proposal requests. propose_requests: Option, /// Handoff responses controlled by the test. handoff_responses: Option, /// Whether mock application proposal requests should remain pending. stall_proposals: bool, /// Whether the mock application drops proposal responses. drop_proposals: bool, /// Decision the mock application uses to answer handoff proposal requests. handoff: Handoff<()>, /// Receives the voter's task handle so a test can abort it and restart the voter. actor_handle: Option>>>>, /// Views whose verification requests reached the mock application. verify_requests: Option>>>, /// Whether every mock application verification should fail. fail_verification: bool, certifier: mocks::application::Certifier, } impl Default for VoterOptions { /// A short leader timeout with certification and retry timeouts far /// beyond test duration, so only leader timeouts fire. fn default() -> Self { Self { leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), local_index: 0, propose_latency_ms: 1.0, verify_latency_ms: 1.0, certify_latency_ms: 1.0, propose_requests: None, handoff_responses: None, stall_proposals: false, drop_proposals: false, handoff: Handoff::Wait, actor_handle: None, verify_requests: None, fail_verification: false, certifier: mocks::application::Certifier::Always, } } } /// Expect the replay marker before any live certificate. async fn expect_replayed>( receiver: &mut mailbox::Receiver>, ) { assert!(matches!( receiver.recv().await.unwrap(), resolver::MailboxMessage::Replayed { .. } )); } /// Helper to set up a voter actor for tests. async fn setup_voter( context: &E, oracle: &commonware_p2p::simulated::Oracle, participants: &[S::PublicKey], schemes: &[S], elector: L, options: VoterOptions, ) -> ( Mailbox, mailbox::Receiver>, mailbox::Receiver>, Arc>, mocks::reporter::Reporter, ) where E: Clock + CryptoRng + Spawner + Storage + Metrics, S: Scheme, L: elector::Config, { let signing = schemes[options.local_index].clone(); let me = participants[options.local_index].clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.to_vec().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let elector = elector.build(signing.participants()); let propose_requests = options.propose_requests; let handoff_responses = options.handoff_responses; let verify_requests = options.verify_requests; let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (options.propose_latency_ms, 0.0), verify_latency: (options.verify_latency_ms, 0.0), certify_latency: (options.certify_latency_ms, 0.0), should_certify: options.certifier, }; let (mut actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); actor.set_stall_proposals(options.stall_proposals); actor.set_drop_proposals(options.drop_proposals); actor.set_handoff(options.handoff); actor.set_fail_verification(options.fail_verification); if let Some(propose_requests) = propose_requests { actor.set_propose_observer(Box::new(move |context| { propose_requests.lock().push(context); })); } if let Some(handoff_responses) = handoff_responses { actor.set_handoff_controller(Box::new(move |_, proposal, response| { handoff_responses.lock().push((proposal, response)); })); } if let Some(verify_requests) = verify_requests { actor.set_verify_observer(Box::new(move |context, _| { verify_requests.lock().push(context.view()); })); } actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_test_{me}"), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: options.leader_timeout, certification_timeout: options.certification_timeout, timeout_retry: options.timeout_retry, skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(16)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, certificate_sender, ); if let Some(slot) = options.actor_handle { *slot.lock() = Some(handle); } ( mailbox, batcher_receiver, resolver_receiver, relay, reporter, ) } /// Certifies view 1 and drives the local validator at `local_index` through its notarize /// vote for view 2, leaving the outgoing tip uncertified for handoff tests. async fn prepare_pipelined_handoff_tip( schemes: &[ed25519::Scheme], local_index: usize, outgoing: &PublicKey, voter_mailbox: &mut Mailbox, batcher_receiver: &mut mailbox::Receiver>, resolver_receiver: &mut mailbox::Receiver< resolver::MailboxMessage, >, relay: &Arc>, ) -> Proposal { let epoch = Epoch::new(333); let quorum = quorum(u32::try_from(schemes.len()).expect("test fixture must fit u32")); assert!(matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Update { .. } )); let genesis = mocks::application::genesis::(epoch); let proposal_1 = Proposal::new( Round::new(epoch, View::new(1)), View::zero(), Sha256::hash(&[b"pipelined_test_view_1"]), ); relay.broadcast( outgoing, Recipients::All, ( proposal_1.payload, (proposal_1.round, genesis, 0u64).encode(), ), ); let (_, notarization_1) = build_notarization(&peers(schemes, local_index), &proposal_1, quorum); voter_mailbox.recovered(Certificate::Notarization(notarization_1)); let mut certified_view_1 = false; let mut entered_view_2 = false; while !certified_view_1 || !entered_view_2 { select! { message = batcher_receiver.recv() => { if matches!( message.unwrap(), batcher::Message::Update { current, .. } if current == View::new(2) ) { entered_view_2 = true; } }, message = resolver_receiver.recv() => { if matches!( message.unwrap(), MailboxMessage::Certified { notarization, success: true, .. } if notarization.view() == View::new(1) ) { certified_view_1 = true; } }, } } let proposal_2 = Proposal::new( Round::new(epoch, View::new(2)), View::new(1), Sha256::hash(&[b"pipelined_test_view_2"]), ); relay.broadcast( outgoing, Recipients::All, ( proposal_2.payload, (proposal_2.round, proposal_1.payload, 1u64).encode(), ), ); voter_mailbox.proposal(proposal_2.clone()); loop { if matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == View::new(2) ) { return proposal_2; } } } /// Waits until the application logs a request whose view, as projected by `view_of`, is /// `view`. async fn wait_for_request( context: &deterministic::Context, requests: &Arc>>, view: View, view_of: impl Fn(&T) -> View, ) { let deadline = context.current() + Duration::from_secs(1); while !requests .lock() .iter() .any(|request| view_of(request) == view) { if context.current() >= deadline { panic!("application did not receive request for {view}"); } context.sleep(Duration::from_millis(1)).await; } } /// Returns the (view, parent) pairs of the logged proposal requests. fn request_parents(requests: &ProposeRequests) -> Vec<(View, View)> { requests .lock() .iter() .map(|request| (request.view(), request.parent.0)) .collect() } async fn take_proposal_response( context: &deterministic::Context, responses: &Arc>>, ) -> T { loop { if let Some(response) = responses.lock().pop() { return response; } context.sleep(Duration::from_millis(1)).await; } } async fn take_certification_request( context: &deterministic::Context, requests: &CertificationRequests, view: View, ) -> oneshot::Sender { loop { if let Some(response) = { let mut requests = requests.lock(); requests .iter() .position(|(request_view, _)| *request_view == view) .map(|index| requests.swap_remove(index).1) } { return response; } context.sleep(Duration::from_millis(1)).await; } } /// Certifies every view but `view`, whose request it parks in `requests` for /// [`take_certification_request`]. fn hold(view: View, requests: CertificationRequests) -> Certifier { Certifier::Controlled(Box::new(move |round, _, response| { if round.view() == view { requests.lock().push((round.view(), response)); } else { response.send(true).unwrap(); } })) } fn trace_position( events: &RecordedEvents, content: &str, span_name: &str, view: View, ) -> usize { let view = view.to_string(); events .iter() .position(|event| { event.metadata.content == content && event .expect_span_at_index(0, |span| { span.expect_content_exact(span_name)?; span.expect_field_exact("view", &view) }) .is_ok() }) .unwrap_or_else(|| panic!("missing {content} trace for view {view}")) } async fn seed_voter_journal( context: deterministic::Context, partition: String, page_cache: CacheRef, scheme: &S, finalization: Finalization, ) where S: Scheme, { seed_voter_journal_artifacts( context, partition, page_cache, scheme, finalization.view(), vec![Artifact::Finalization(finalization)], ) .await; } async fn seed_voter_journal_artifacts( context: deterministic::Context, partition: String, page_cache: CacheRef, scheme: &S, view: View, artifacts: Vec>, ) where S: Scheme, { let mut journal = Journal::<_, Artifact>::init( context.child("journal"), JConfig { partition, compression: None, codec_config: scheme.certificate_codec_config(), page_cache, write_buffer: NZUsize!(1024 * 1024), }, ) .await .expect("unable to initialize voter journal"); for artifact in artifacts { assert_eq!(artifact.view(), view); (journal, _, _) = journal .append(view.get(), &artifact) .await .expect("unable to append voter journal artifact"); } journal .sync(view.get()) .await .expect("unable to sync voter journal"); } struct VoterFloorStart where S: Scheme, { partition: String, epoch: Epoch, floor: Floor, skip_budget: u64, page_cache: CacheRef, certifier: mocks::application::Certifier, } async fn start_voter_with_floor( context: &mut deterministic::Context, oracle: &Oracle, participants: &[S::PublicKey], schemes: &[S], elector: L, start: VoterFloorStart, ) -> ( Mailbox, mailbox::Receiver>, mailbox::Receiver>, Arc>, mocks::reporter::Reporter, Handle<()>, ) where S: Scheme, L: elector::Config, { let VoterFloorStart { partition, epoch, floor, skip_budget, page_cache, certifier, } = start; let me = participants[0].clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.to_vec().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let elector = elector.build(schemes[0].participants()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: certifier, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application, reporter: reporter.clone(), partition, epoch, floor, mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(6), timeout_retry: Duration::from_mins(60), skip_budget, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache, }; let (voter, mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _) = oracle.control(me).register(1, TEST_QUOTA).await.unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, certificate_sender, ); ( mailbox, batcher_receiver, resolver_receiver, relay, reporter, handle, ) } async fn expect_restarted_voter_at_floor( context: &mut deterministic::Context, batcher_receiver: &mut mailbox::Receiver>, resolver_receiver: &mut mailbox::Receiver>, finalized: View, ) where S: Scheme, { let mut saw_update = false; let mut saw_finalization = false; loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, finalized: found, .. } = msg.expect("batcher mailbox closed") { assert_eq!(current, finalized.next()); assert_eq!(found, finalized); saw_update = true; } }, msg = resolver_receiver.recv() => { if let resolver::MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } = msg.expect("resolver mailbox closed") { let view = finalization.view(); assert!( view <= finalized, "restarted voter reported finalization above expected floor: {view}" ); saw_finalization |= view == finalized; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("timed out waiting for restarted voter at finalized view {finalized}"); }, } if saw_update && saw_finalization { break; } } } #[test_traced("WARN")] fn test_voter_restart_newer_floor_skips_stale_journal() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_restart_newer_floor".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let partition = "voter_restart_newer_floor".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Persist a stale finalization that should be ignored once the // configured floor is applied before replay. let old_view = View::new(3); let old_proposal = Proposal::new( Round::new(epoch, old_view), old_view.previous().unwrap(), Sha256::hash(&[b"old-journal-finalization"]), ); let (_, old_finalization) = build_finalization(&schemes, &old_proposal, quorum); seed_voter_journal( context.child("seed_journal"), partition.clone(), page_cache.clone(), &schemes[0], old_finalization, ) .await; // Start from a newer finalized floor over the same journal partition. let floor_view = View::new(8); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"newer-floor-finalization"]), ); let (_, floor_finalization) = build_finalization(&schemes, &floor_proposal, quorum); let (_mailbox, mut batcher_receiver, mut resolver_receiver, _relay, reporter, _handle) = start_voter_with_floor( &mut context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterFloorStart { partition, epoch, floor: Floor::Finalized(floor_finalization), skip_budget: u64::MAX, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; // The voter should seed its in-memory state from the configured floor // and should not surface the older journal finalization. expect_restarted_voter_at_floor( &mut context, &mut batcher_receiver, &mut resolver_receiver, floor_view, ) .await; let finalizations = reporter.finalizations.lock(); assert!(finalizations.contains_key(&floor_view)); assert!( !finalizations.contains_key(&old_view), "stale journal finalization must be skipped when the configured floor is newer" ); }); } /// A voter restarted from a finalized floor reports the floor finalization /// once at startup. A later certificate at the floor view must not report /// or broadcast it again. #[test_traced("WARN")] fn test_voter_restart_floor_finalization_not_resent() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_restart_floor_not_resent".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Observe the certificates the voter broadcasts. let observer = participants[1].clone(); let (_, mut certificates) = oracle .control(observer.clone()) .register(1, TEST_QUOTA) .await .unwrap(); oracle .add_link( participants[0].clone(), observer, Link { latency: Duration::ZERO, jitter: Duration::ZERO, success_rate: probability!(1.0), }, ) .await .unwrap(); // Start from a finalized floor at view 8. let finalization = |view: View| { let proposal = Proposal::new( Round::new(epoch, view), view.previous().unwrap(), Sha256::hash(&[&view.get().to_be_bytes()]), ); build_finalization(&schemes, &proposal, quorum).1 }; let floor_view = View::new(8); let floor_finalization = finalization(floor_view); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); let ( mut mailbox, mut batcher_receiver, mut resolver_receiver, _relay, reporter, _handle, ) = start_voter_with_floor( &mut context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterFloorStart { partition: "voter_restart_floor_not_resent".to_string(), epoch, floor: Floor::Finalized(floor_finalization.clone()), skip_budget: u64::MAX, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; expect_restarted_voter_at_floor( &mut context, &mut batcher_receiver, &mut resolver_receiver, floor_view, ) .await; // A peer forwards the floor finalization again, followed by a new // finalization that ends the observation. let next_view = floor_view.next(); mailbox.recovered(Certificate::Finalization(floor_finalization)); mailbox.recovered(Certificate::Finalization(finalization(next_view))); // Only the new finalization is broadcast, reported to the // application, and reported to the resolver. loop { let (_, encoded) = certificates.recv().await.unwrap(); let certificate: Certificate = Certificate::decode_cfg(encoded, &schemes[0].certificate_codec_config()) .unwrap(); assert_ne!( certificate.view(), floor_view, "floor finalization broadcast again" ); if certificate.view() == next_view { break; } } { let finalizations = reporter.finalizations.lock(); assert_eq!(finalizations[&floor_view].1, 1); assert_eq!(finalizations[&next_view].1, 1); } loop { if let resolver::MailboxMessage::Updated { certificate, .. } = resolver_receiver.recv().await.unwrap() { assert_ne!( certificate.view(), floor_view, "floor finalization reported again" ); if certificate.view() == next_view { break; } } } }); } #[test_traced("WARN")] fn test_voter_restart_older_floor_ignored_for_newer_journal() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_restart_older_floor".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let partition = "voter_restart_older_floor".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Persist a newer finalization so replay can advance past the // configured floor. let journal_view = View::new(8); let journal_proposal = Proposal::new( Round::new(epoch, journal_view), journal_view.previous().unwrap(), Sha256::hash(&[b"newer-journal-finalization"]), ); let (_, journal_finalization) = build_finalization(&schemes, &journal_proposal, quorum); seed_voter_journal( context.child("seed_journal"), partition.clone(), page_cache.clone(), &schemes[0], journal_finalization, ) .await; // Provide an older floor; journal replay should supersede it. let floor_view = View::new(3); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"older-floor-finalization"]), ); let (_, floor_finalization) = build_finalization(&schemes, &floor_proposal, quorum); let (_mailbox, mut batcher_receiver, mut resolver_receiver, _relay, reporter, _handle) = start_voter_with_floor( &mut context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterFloorStart { partition, epoch, floor: Floor::Finalized(floor_finalization), skip_budget: u64::MAX, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; // The configured floor may be reported before replay, but the // replayed finalization should become the active floor. expect_restarted_voter_at_floor( &mut context, &mut batcher_receiver, &mut resolver_receiver, journal_view, ) .await; let finalizations = reporter.finalizations.lock(); assert!(finalizations.contains_key(&journal_view)); assert!( finalizations.contains_key(&floor_view), "configured floor should be reported before journal replay supersedes it" ); }); } #[test_traced("WARN")] fn test_voter_replay_floor_does_not_move_during_replay() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_replay_floor_static".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let partition = "voter_replay_floor_static".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); let floor_view = View::new(3); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"static-replay-floor"]), ); let (_, floor_finalization) = build_finalization(&schemes, &floor_proposal, quorum); // Place two above-floor artifacts in the same journal section. The // first one advances `last_finalized`, but the second must still // replay because it is above the configured floor. let journal_view = View::new(8); let journal_proposal = Proposal::new( Round::new(epoch, journal_view), journal_view.previous().unwrap(), Sha256::hash(&[b"same-section-finalize"]), ); let (mut finalizes, journal_finalization) = build_finalization(&schemes, &journal_proposal, quorum); let finalize = finalizes.remove(0); let finalized_payload = finalize.proposal.payload; seed_voter_journal_artifacts( context.child("seed_journal"), partition.clone(), page_cache.clone(), &schemes[0], journal_view, vec![ Artifact::Finalization(journal_finalization), Artifact::Finalize(finalize), ], ) .await; let (_mailbox, mut batcher_receiver, mut resolver_receiver, _relay, reporter, _handle) = start_voter_with_floor( &mut context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterFloorStart { partition, epoch, floor: Floor::Finalized(floor_finalization), skip_budget: u64::MAX, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; expect_restarted_voter_at_floor( &mut context, &mut batcher_receiver, &mut resolver_receiver, journal_view, ) .await; let finalizes = reporter.finalizes.lock(); let signers = finalizes .get(&journal_view) .and_then(|payloads| payloads.get(&finalized_payload)) .expect("above-floor finalize vote should replay"); assert!(signers.contains(&participants[0])); }); } /// Helper to advance to a specific view by sending a finalization for the previous view. async fn advance_to_view>( mailbox: &mut Mailbox, batcher_receiver: &mut mailbox::Receiver>, schemes: &[S], quorum: u32, target: View, ) -> Sha256Digest { let prev_view = target.previous().expect("target view must be > 0"); let payload = Sha256::hash(&[&prev_view.get().to_be_bytes()]); let proposal = Proposal::new( Round::new(Epoch::new(333), prev_view), prev_view.previous().unwrap_or(View::zero()), payload, ); let (_, finalization) = build_finalization(schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for target view update loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, .. } => { if current < target { continue; } assert_eq!(current, target); break; } batcher::Message::Constructed(_) => {} } } payload } /// Trigger processing of an uninteresting view from the resolver after /// jumping ahead to a new finalize view: /// /// 1. Send a finalization for view 100. /// 2. Send a notarization from resolver for view 50 (should be ignored). /// 3. Send a finalization for view 300 (should be processed). fn stale_backfill(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers( context.child("network"), participants.clone(), false, ) .await; // Initialize voter actor let me = participants[0].clone(); let reporter_config = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_config); let elector = elector.build(schemes[0].participants()); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (10.0, 5.0), verify_latency: (10.0, 5.0), certify_latency: (10.0, 5.0), should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("application"), application_cfg); actor.start(); let cfg = Config { scheme: schemes[0].clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(10), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NonZeroUsize::new(1024 * 1024).unwrap(), write_buffer: NonZeroUsize::new(1024 * 1024).unwrap(), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (actor, mut mailbox) = Actor::new(context.child("actor"), cfg); // Create a dummy resolver mailbox let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(10)); let resolver = resolver::Mailbox::new(resolver_sender); // Create a dummy batcher mailbox let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(1024)); let batcher = batcher::Mailbox::new(batcher_sender); // Create network senders for broadcasting votes and certificates let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); // Run the actor actor.start(batcher, resolver, vote_sender, certificate_sender); // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Send finalization via voter mailbox (view 100) let payload = Sha256::hash(&[b"test"]); let proposal = Proposal::new( Round::new(Epoch::new(333), View::new(100)), View::new(50), payload, ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for batcher to be notified loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(101)); assert_eq!(finalized, View::new(100)); break; } _ => { continue; } } } // Wait for resolver to be notified expect_replayed(&mut resolver_receiver).await; let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization.view(), View::new(100)); } _ => panic!("unexpected resolver message"), } // Send old notarization from resolver that should be ignored (view 50) let payload = Sha256::hash(&[b"test2"]); let proposal = Proposal::new( Round::new(Epoch::new(333), View::new(50)), View::new(49), payload, ); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Send new finalization via voter mailbox (view 300) let payload = Sha256::hash(&[b"test3"]); let proposal = Proposal::new( Round::new(Epoch::new(333), View::new(300)), View::new(100), payload, ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for batcher to be notified loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(301)); assert_eq!(finalized, View::new(300)); break; } _ => { continue; } } } // Wait for resolver to be notified let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization.view(), View::new(300)); } _ => panic!("unexpected resolver message"), } }); } #[test_traced] fn test_stale_backfill() { stale_backfill::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); stale_backfill::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); stale_backfill::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); stale_backfill::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); stale_backfill::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); stale_backfill::<_, _, RoundRobin>(secp256r1::fixture, RoundRobin::default()); } /// Process an interesting view below the oldest tracked view: /// /// 1. Advance last_finalized to a view 50. /// 2. Ensure self.views contains a view V_A (45) which is interesting, /// and becomes the 'oldest' view when prune_views runs, setting the journal floor. /// Crucially, ensure there's a "gap" so that V_A is not LF - view_retention. /// 3. Let prune_views run, setting the journal floor to V_A. /// 4. Inject a message for V_B such that V_B < V_A but V_B is still "interesting" /// relative to the current last_finalized. fn append_old_interesting_view(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"test_prune_panic".to_vec(); let view_retention = ViewDelta::new(10); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup the target Voter actor (validator 0) let signing = schemes[0].clone(); let me = participants[0].clone(); let reporter_config = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_config); let elector = elector.build(signing.participants()); let relay = Arc::new(mocks::relay::Relay::::new()); let app_config = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("app"), app_config); actor.start(); let voter_config = Config { scheme: signing.clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_actor_test_{me}"), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_millis(1000), timeout_retry: Duration::from_millis(1000), skip_budget: u64::MAX, view_retention, replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (actor, mut mailbox) = Actor::new(context.child("actor"), voter_config); // Create a dummy resolver mailbox let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(10)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); // Create a dummy batcher mailbox let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(10)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); // Create network senders for broadcasting votes and certificates let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); // Start the actor actor.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Establish Prune Floor (50 - 10 + 5 = 45) // // Theoretical retention floor is 50-10 = 40. // We want journal pruned at 45. let lf_target = View::new(50); let journal_floor_target = lf_target .saturating_sub(view_retention) .saturating_add(ViewDelta::new(5)); // Send Finalization via voter mailbox to advance last_finalized let proposal_lf = Proposal::new( Round::new(Epoch::new(333), lf_target), lf_target.previous().unwrap(), Sha256::hash(&[b"test"]), ); let (_, finalization) = build_finalization(&schemes, &proposal_lf, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for batcher to be notified loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(51)); assert_eq!(finalized, View::new(50)); break; } _ => { continue; } } } expect_replayed(&mut resolver_receiver).await; // Wait for resolver to be notified let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization.view(), View::new(50)); } _ => panic!("unexpected resolver message"), } // Send a Notarization for `journal_floor_target` to ensure it's in `actor.views` let proposal_jft = Proposal::new( Round::new(Epoch::new(333), journal_floor_target), journal_floor_target.previous().unwrap(), Sha256::hash(&[b"test2"]), ); let (_, notarization_for_floor) = build_notarization(&schemes, &proposal_jft, quorum); mailbox.recovered(Certificate::Notarization(notarization_for_floor)); // Wait for resolver to be notified let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Notarization(notarization), .. } => { assert_eq!(notarization.view(), journal_floor_target); } _ => panic!("unexpected resolver message"), } // Send notarization below oldest tracked view (42) // // problematic_view (42) < journal_floor_target (45) // Viewport::retains(42): 42 >= floor (LF(50) - AT(10) = 40) let problematic_view = journal_floor_target.saturating_sub(ViewDelta::new(3)); let proposal_bft = Proposal::new( Round::new(Epoch::new(333), problematic_view), problematic_view.previous().unwrap(), Sha256::hash(&[b"test3"]), ); let (_, notarization_for_bft) = build_notarization(&schemes, &proposal_bft, quorum); mailbox.recovered(Certificate::Notarization(notarization_for_bft)); // Wait for resolver to be notified let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Notarization(notarization), .. } => { assert_eq!(notarization.view(), problematic_view); } _ => panic!("unexpected resolver message"), } // Send Finalization via voter mailbox to new view (100) let proposal_lf = Proposal::new( Round::new(Epoch::new(333), View::new(100)), View::new(99), Sha256::hash(&[b"test4"]), ); let (_, finalization) = build_finalization(&schemes, &proposal_lf, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for batcher to be notified loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(101)); assert_eq!(finalized, View::new(100)); break; } _ => { continue; } } } // Wait for resolver to be notified let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization.view(), View::new(100)); } _ => panic!("unexpected resolver message"), } }); } #[test_traced] fn test_append_old_interesting_view() { append_old_interesting_view::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); append_old_interesting_view::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); append_old_interesting_view::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); append_old_interesting_view::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); append_old_interesting_view::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); append_old_interesting_view::<_, _, RoundRobin>(secp256r1::fixture, RoundRobin::default()); } /// Test that voter can process finalization from batcher without notarization. fn finalization_without_notarization_certificate(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"finalization_without_notarization".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let (mut mailbox, mut batcher_receiver, mut resolver_receiver, _, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions::default(), ) .await; // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Build a finalization certificate (without notarization) let view = View::new(2); let proposal = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"finalize_without_notarization"]), ); let (_, expected_finalization) = build_finalization(&schemes, &proposal, quorum); // Send finalization certificate via voter mailbox mailbox.recovered(Certificate::Finalization(expected_finalization.clone())); // Wait for the actor to report the finalization let mut finalized_view = None; while let Some(message) = resolver_receiver.recv().await { match message { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { finalized_view = Some(finalization.view()); break; } _ => continue, } } assert_eq!(finalized_view, Some(view)); // Wait for a finalization to be recorded loop { { let finalizations = reporter.finalizations.lock(); // Finalization must match the signatures recovered from finalize votes if matches!( finalizations.get(&view), Some((finalization, _)) if finalization == &expected_finalization ) { break; } } context.sleep(Duration::from_millis(10)).await; } // Verify no notarization certificate was recorded let notarizations = reporter.notarizations.lock(); assert!(notarizations.is_empty()); }); } #[test_traced] fn test_finalization_without_notarization_certificate() { finalization_without_notarization_certificate::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); finalization_without_notarization_certificate::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); finalization_without_notarization_certificate::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); finalization_without_notarization_certificate::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); finalization_without_notarization_certificate::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); finalization_without_notarization_certificate::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } fn certificate_conflicts_proposal(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"certificate_conflicts_proposal_test".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let (mut mailbox, mut batcher_receiver, mut resolver_receiver, _, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions::default(), ) .await; // Wait for initial batcher notification let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Send proposal A from batcher (simulating leader's proposal being forwarded) let view = View::new(2); let proposal_a = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal_a"]), ); mailbox.proposal(proposal_a.clone()); // Give it time to process the proposal context.sleep(Duration::from_millis(10)).await; // Send notarization certificate for a DIFFERENT proposal B let proposal_b = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal_b"]), ); let (_, notarization_b) = build_notarization(&schemes, &proposal_b, quorum); mailbox.recovered(Certificate::Notarization(notarization_b.clone())); // Verify the certificate was accepted expect_replayed(&mut resolver_receiver).await; let msg = resolver_receiver .recv() .await .expect("failed to receive resolver message"); match msg { MailboxMessage::Updated { certificate: Certificate::Notarization(notarization), .. } => { assert_eq!(notarization.proposal, proposal_b); assert_eq!(notarization, notarization_b); } _ => panic!("unexpected resolver message"), } // Wait for notarization B to be recorded (not A) loop { { let notarizations = reporter.notarizations.lock(); if matches!( notarizations.get(&view), Some(notarization) if notarization == ¬arization_b ) { break; } } context.sleep(Duration::from_millis(10)).await; } // Ensure no finalize vote is broadcast (don't vote on conflict) context.sleep(Duration::from_millis(100)).await; loop { let Some(Some(message)) = batcher_receiver.recv().now_or_never() else { break; }; match message { batcher::Message::Constructed(Vote::Finalize(_)) => { panic!("finalize vote should not be broadcast"); } batcher::Message::Update { .. } => {} _ => continue, } } }); } #[test_traced] fn test_certificate_conflicts_proposal() { certificate_conflicts_proposal::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certificate_conflicts_proposal::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certificate_conflicts_proposal::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certificate_conflicts_proposal::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certificate_conflicts_proposal::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); certificate_conflicts_proposal::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } fn proposal_conflicts_certificate(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"proposal_conflicts_certificate_test".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let (mut mailbox, mut batcher_receiver, mut resolver_receiver, _, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions::default(), ) .await; // Wait for initial batcher notification let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { .. } => {} _ => panic!("unexpected batcher message"), } let view = View::new(2); let proposal_a = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal_a"]), ); let proposal_b = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal_b"]), ); // Send certificate for proposal A FIRST let (_, notarization_a) = build_notarization(&schemes, &proposal_a, quorum); mailbox.recovered(Certificate::Notarization(notarization_a.clone())); // Verify the certificate was accepted expect_replayed(&mut resolver_receiver).await; let msg = resolver_receiver.recv().await.unwrap(); match msg { MailboxMessage::Updated { certificate: Certificate::Notarization(notarization), .. } => { assert_eq!(notarization.proposal, proposal_a); } _ => panic!("unexpected resolver message"), } // Wait for notarization A to be recorded loop { { let notarizations = reporter.notarizations.lock(); if matches!( notarizations.get(&view), Some(notarization) if notarization == ¬arization_a ) { break; } } context.sleep(Duration::from_millis(10)).await; } // Ensure finalize vote is sent loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Constructed(Vote::Finalize(finalize)) => { assert_eq!( finalize.proposal, proposal_a, "finalize should be for certificate's proposal A" ); break; } batcher::Message::Update { .. } => {} _ => context.sleep(Duration::from_millis(10)).await, } } // Now send proposal B from batcher mailbox.proposal(proposal_b.clone()); // Wait for proposal B to be recorded (no issue) context.sleep(Duration::from_millis(100)).await; }); } #[test_traced] fn test_proposal_conflicts_certificate() { proposal_conflicts_certificate::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); proposal_conflicts_certificate::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); proposal_conflicts_certificate::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); proposal_conflicts_certificate::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); proposal_conflicts_certificate::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); proposal_conflicts_certificate::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } fn certificate_verifies_proposal(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"certificate_conflicts_proposal_test".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: participants[0].clone(), propose_latency: (1.0, 0.0), verify_latency: (100_000.0, 0.0), // Very slow verification certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(participants[0].clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "voter_certificate_verifies_proposal_test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); let me = participants[0].clone(); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for initial batcher notification let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { .. } => {} _ => panic!("unexpected batcher message"), } let view = View::new(2); let proposal = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"same_proposal"]), ); // Send proposal from batcher first mailbox.proposal(proposal.clone()); // Give it time to start verification (but it won't complete due to slow latency) context.sleep(Duration::from_millis(10)).await; // Send certificate for the SAME proposal let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization.clone())); expect_replayed(&mut resolver_receiver).await; // The certificate should verify the proposal immediately let msg = resolver_receiver.recv().await.unwrap(); match msg { MailboxMessage::Updated { certificate: Certificate::Notarization(n), .. } => { assert_eq!(n.proposal, proposal); } _ => panic!("unexpected resolver message"), } // Wait for notarization to be recorded loop { { let notarizations = reporter.notarizations.lock(); if matches!( notarizations.get(&view), Some(n) if n == ¬arization ) { break; } } context.sleep(Duration::from_millis(10)).await; } // Should be able to finalize since the proposal was verified by the certificate loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Constructed(Vote::Finalize(finalize)) => { assert_eq!(finalize.proposal, proposal); break; } batcher::Message::Update { .. } => {} _ => context.sleep(Duration::from_millis(10)).await, } } }); } #[test_traced] fn test_certificate_verifies_proposal() { certificate_verifies_proposal::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certificate_verifies_proposal::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certificate_verifies_proposal::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certificate_verifies_proposal::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certificate_verifies_proposal::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); certificate_verifies_proposal::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Test that our proposal is dropped when it conflicts with a peer's notarize vote. /// /// This is a regression test for a byzantine scenario where multiple nodes share the /// same signing key: /// 1. A peer with our identity sends a notarize vote for proposal A /// 2. Our automaton completes with a different proposal B /// 3. Our proposal should be dropped when the conflict is detected fn drop_our_proposal_on_conflict(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"drop_our_proposal_on_conflict_test".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Figure out who the leader will be for view 2 let view2_round = Round::new(epoch, View::new(2)); let elector_config = RoundRobin::::default(); let temp_elector: RoundRobinElector = elector_config.clone().build(schemes[0].participants()); let leader_idx = temp_elector.elect(view2_round, ()); let leader = participants[usize::from(leader_idx)].clone(); // Create a voter with the leader's identity let leader_scheme = schemes[usize::from(leader_idx)].clone(); // Setup application mock with some latency so we can inject peer // message before automaton completes let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: leader.clone(), propose_latency: (50.0, 10.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); actor.start(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: leader_scheme.clone(), elector: elector_config.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); // Initialize voter actor let voter_cfg = Config { scheme: leader_scheme.clone(), elector: elector_config.clone().build(leader_scheme.participants()), blocker: oracle.control(leader.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "voter_leader".to_string(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); // Resolver and batcher mailboxes let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); // Register network channels let (vote_sender, _) = oracle .control(leader.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(leader.clone()) .register(1, TEST_QUOTA) .await .unwrap(); // Start the voter voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for initial batcher notification let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Now create a finalization certificate for view 1 to advance to view 2 let view1_round = Round::new(epoch, View::new(1)); let view1_proposal = Proposal::new(view1_round, View::new(0), Sha256::hash(&[b"view1_payload"])); let (_, finalization) = build_finalization(&schemes, &view1_proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for batcher to be notified loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(2)); assert_eq!(finalized, View::new(1)); break; } _ => { continue; } } } // Wait a bit for the voter to request a proposal from automaton for view 2 context.sleep(Duration::from_millis(5)).await; // Create a conflicting proposal from ourselves (equivocating) for view 2 let conflicting_proposal = Proposal::new( view2_round, View::new(1), Sha256::hash(&[b"leader_proposal"]), ); // Send the proposal via mailbox (simulating batcher receiving leader's notarize) // This happens AFTER we requested a proposal but BEFORE the automaton responds mailbox.proposal(conflicting_proposal.clone()); // Ensure we construct a notarize for our proposal while let Ok(message) = batcher_receiver.try_recv() { match message { batcher::Message::Constructed(Vote::Notarize(notarize)) => { assert!(notarize.proposal == conflicting_proposal); } _ => panic!("unexpected batcher message"), } } // Now wait for our automaton to complete its proposal // This should trigger `our_proposal` which will see the conflicting proposal context.sleep(Duration::from_millis(100)).await; // Add a notarization certificate for conflicting proposal let (_, conflicting_notarization) = build_notarization(&schemes, &conflicting_proposal, quorum); mailbox.recovered(Certificate::Notarization(conflicting_notarization.clone())); // Wait for a finalize vote to be broadcast (we drop our own conflicting proposal rather than marking as replaced) loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Constructed(Vote::Finalize(f)) => { assert_eq!(f.proposal, conflicting_proposal); break; } batcher::Message::Update { .. } => {} _ => context.sleep(Duration::from_millis(10)).await, } } }); } #[test_traced] fn test_drop_our_proposal_on_conflict() { drop_our_proposal_on_conflict(bls12381_threshold_vrf::fixture::); drop_our_proposal_on_conflict(bls12381_threshold_vrf::fixture::); drop_our_proposal_on_conflict(bls12381_multisig::fixture::); drop_our_proposal_on_conflict(bls12381_multisig::fixture::); drop_our_proposal_on_conflict(ed25519::fixture); drop_our_proposal_on_conflict(secp256r1::fixture); } fn populate_resolver_on_restart(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"populate_resolver_on_restart_test".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: participants[0].clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); actor.start(); // Initialize voter actor let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(participants[0].clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "voter_populate_resolver_on_restart_test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); // Resolver and batcher mailboxes let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); // Register network channels for the validator let me = participants[0].clone(); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); // Start the actor let handle = voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } expect_replayed(&mut resolver_receiver).await; // Build finalization certificate for view 2 let view = View::new(2); let proposal = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"finalize_without_notarization"]), ); let (_, expected_finalization) = build_finalization(&schemes, &proposal, quorum); // Send finalization certificate via voter mailbox mailbox.recovered(Certificate::Finalization(expected_finalization.clone())); // Wait for finalization to be sent to resolver let finalization = resolver_receiver.recv().await.unwrap(); match finalization { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization, expected_finalization); } _ => panic!("unexpected resolver message"), } // Restart voter handle.abort(); // Initialize voter actor let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(participants[0].clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "voter_populate_resolver_on_restart_test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); // Resolver and batcher mailboxes let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); // Register new network channels for the validator (we don't use p2p, so this doesn't matter) let me = participants[0].clone(); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); // Start the actor voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(3)); assert_eq!(finalized, View::new(2)); } _ => panic!("unexpected batcher message"), } // Wait for finalization to be sent to resolver let finalization = resolver_receiver.recv().await.unwrap(); match finalization { MailboxMessage::Updated { certificate: Certificate::Finalization(finalization), .. } => { assert_eq!(finalization, expected_finalization); } _ => panic!("unexpected resolver message"), } }); } /// A replayed certificate can precede the floor that covers it, such as a /// later finalization or its own successful certification. A failed /// certification likewise precedes the nullification that covers its view. /// Restart must not request nullifications that the recovered floor or a /// journaled nullification covers, but must still request the gaps above /// the floor. #[test_traced("WARN")] fn test_restart_replay_fetches_only_above_recovered_floor() { // (term length, notarized view, journal notarization with its certification verdict, // journal finalization, nullified view, expected fetches) let cases = [ ( TermLength::ONE, 10_000, Some(true), true, 10_004, vec![10_002, 10_003], ), ( TermLength::ONE, 10_000, Some(true), false, 10_003, vec![10_001, 10_002], ), (TermLength::ONE, 10_000, None, false, 3, vec![1, 2]), (TermLength::ONE, 3, Some(false), false, 3, vec![1, 2]), ( TermLength::new(NZU32!(5)), 10_001, Some(true), true, 10_004, vec![10_003], ), (TermLength::ONE, 10_000, None, true, 10_000, vec![]), ]; for (term_length, notarized, verdict, with_finalization, nullified, expected) in cases { let runner = deterministic::Runner::timed(Duration::from_secs(30)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"replay_floor_catchup", n); let elector = if term_length == TermLength::ONE { RoundRobin::::default() } else { RoundRobin::::default().with_term( term_length, Duration::from_secs(20), ViewDelta::new(0), ) }; let notarized = View::new(notarized); let nullified = View::new(nullified); let mut certificates = Vec::new(); if verdict.is_some() { let proposal = Proposal::new( Round::new(epoch, notarized), notarized.previous().unwrap(), Sha256::hash(&[b"replay-notarized"]), ); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); certificates.push(Certificate::Notarization(notarization)); } let mut finalized = View::zero(); if with_finalization { finalized = notarized.next(); let proposal = Proposal::new( Round::new(epoch, finalized), notarized, Sha256::hash(&[b"replay-finalized"]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); certificates.push(Certificate::Finalization(finalization)); } let (_, nullification) = build_nullification(&schemes, Round::new(epoch, nullified), quorum); certificates.push(Certificate::Nullification(nullification)); certificates.sort_by_key(|certificate| certificate.view()); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); let partition = "replay_floor_catchup".to_string(); let start = |partition: String| VoterFloorStart { partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), skip_budget: u64::MAX, page_cache: page_cache.clone(), certifier: mocks::application::Certifier::Custom(Box::new(move |_, _| { verdict.unwrap_or(true) })), }; // Journal each certificate through a live voter. Wait for the // view advance that follows its sync and, for a notarization, the // durable certification verdict before aborting. A notarization // that fails certification does not advance the view, so only its // verdict is awaited. let mut initial = context.child("initial"); let oracle = start_test_network_with_peers( initial.child("network"), participants.clone(), true, ) .await; let (mut mailbox, mut batcher, mut resolver, _, _, handle) = start_voter_with_floor( &mut initial, &oracle, &participants, &schemes, elector.clone(), start(partition.clone()), ) .await; for certificate in &certificates { mailbox.recovered(certificate.clone()); let advanced_to = match certificate { Certificate::Nullification(_) => { certificate.view().term_end(term_length).next() } _ => certificate.view().next(), }; let expected_verdict = match certificate { Certificate::Notarization(_) => verdict, _ => None, }; let mut advanced = expected_verdict == Some(false); let mut certified = expected_verdict.is_none(); while !(advanced && certified) { select! { message = batcher.recv() => { advanced |= matches!( message.unwrap(), batcher::Message::Update { current, .. } if current == advanced_to ); }, message = resolver.recv() => { certified |= matches!( message.unwrap(), MailboxMessage::Certified { notarization, success, .. } if notarization.view() == certificate.view() && Some(success) == expected_verdict ); }, } } } handle.abort(); assert!(handle.await.is_err()); // Apply the restarted voter's resolver messages to resolver state // until every journaled certificate the floor leaves usable has // been forwarded. let mut restarted = context.child("restarted"); let oracle = start_test_network_with_peers( restarted.child("network"), participants.clone(), true, ) .await; let (_mailbox, mut batcher, mut resolver, _, _, handle) = start_voter_with_floor( &mut restarted, &oracle, &participants, &schemes, elector, start(partition), ) .await; // Queue the whole recovered sequence through a full mailbox first, // so the end-of-replay marker must survive the overflow policy. let (sender, mut receiver) = mailbox::new(context.child("replay_overflow"), NZUsize!(1)); let mut state = resolver::State::new(term_length); let mut fetches = Vec::new(); let mut record = |effects: Vec| { fetches.extend(effects.into_iter().filter_map(|effect| match effect { resolver::Effect::Fetch { view, .. } => Some(view.get()), _ => None, })); }; let mut remaining: Vec<_> = certificates .iter() .filter(|certificate| certificate.view() >= finalized) .map(Encode::encode) .collect(); loop { let message = resolver.recv().await.unwrap(); if let MailboxMessage::Updated { certificate, .. } = &message { let encoded = certificate.encode(); remaining.retain(|expected| *expected != encoded); } let replayed = matches!(message, MailboxMessage::Replayed { .. }); assert!(sender.enqueue(message).accepted()); if replayed { break; } } assert!(remaining.is_empty()); while let Ok(message) = receiver.try_recv() { match message { MailboxMessage::Updated { certificate, .. } => { record(state.updated(&certificate)) } MailboxMessage::Certified { notarization, success, .. } => { record(state.certified(notarization, success)); } MailboxMessage::Resolve { .. } => {} MailboxMessage::Replayed { .. } => record(state.replayed()), } } assert_eq!(fetches, expected); let entered = nullified.term_end(term_length).next().max(finalized.next()); assert!(matches!( batcher.recv().await.unwrap(), batcher::Message::Update { current, finalized: found, .. } if current == entered && found == finalized )); handle.abort(); assert!(handle.await.is_err()); }); } } #[test_traced] fn test_populate_resolver_on_restart() { populate_resolver_on_restart::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); populate_resolver_on_restart::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); populate_resolver_on_restart::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); populate_resolver_on_restart::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); populate_resolver_on_restart::<_, _, RoundRobin>(ed25519::fixture, RoundRobin::default()); populate_resolver_on_restart::<_, _, RoundRobin>(secp256r1::fixture, RoundRobin::default()); } /// Regression: startup must consume timeout hints returned by initial batcher update. /// /// On restart, we recover into `target_view` and inject `LeaderNullify` from the /// first `batcher.update`. Even with long timeouts, voter must emit `nullify(target_view)` /// immediately rather than waiting for `leader_timeout`. fn startup_update_timeout_hint_nullifies_recovered_view(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"startup_update_timeout_hint_nullify".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let elector = elector.build(schemes[0].participants()); let relay = Arc::new(mocks::relay::Relay::::new()); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let partition = "voter_startup_update_timeout_hint_nullify".to_string(); let epoch = Epoch::new(333); let make_cfg = |page_cache: CacheRef| Config { scheme: schemes[0].clone(), elector: elector.clone(), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), // Long deadlines prove nullify comes from startup timeout hint, not timer expiry. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache, }; // First run: persist progress to a later view. let cfg = make_cfg(CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE)); let (voter, mut mailbox) = Actor::new(context.child("voter_initial"), cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(32)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, certificate_sender, ); match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => {}, _ => panic!("expected initial update"), } let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; handle.abort(); // Restart and inject startup timeout hint from first update. let cfg = make_cfg(CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE)); let (voter, mut mailbox) = Actor::new(context.child("voter_restarted"), cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(32)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, certificate_sender, ); match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, finalized, .. } => { assert_eq!(current, target_view); assert_eq!(finalized, target_view.previous().unwrap()); mailbox.timeout(Round::new(Epoch::new(333), current), TimeoutReason::LeaderNullify); } _ => panic!("expected startup update after restart"), } // Expect immediate nullify from startup timeout hint despite 10s timeouts. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected immediate nullify for recovered view {target_view} from startup timeout hint" ); }, } } }); } #[test_traced] fn test_startup_update_timeout_hint_nullifies_recovered_view() { startup_update_timeout_hint_nullifies_recovered_view::<_, _>( bls12381_threshold_vrf::fixture::, ); startup_update_timeout_hint_nullifies_recovered_view::<_, _>( bls12381_threshold_vrf::fixture::, ); startup_update_timeout_hint_nullifies_recovered_view::<_, _>( bls12381_multisig::fixture::, ); startup_update_timeout_hint_nullifies_recovered_view::<_, _>( bls12381_multisig::fixture::, ); startup_update_timeout_hint_nullifies_recovered_view::<_, _>(ed25519::fixture); startup_update_timeout_hint_nullifies_recovered_view::<_, _>(secp256r1::fixture); } /// Skip eligibility after replay is derived from durable finalization and /// term progress. Restarting must not restore budget consumed by an unfinalized /// term, while a later finalization must make a pending timeout eligible. #[test_traced("WARN")] fn test_replay_preserves_exhausted_skip_budget() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_replay_skip_budget".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let partition = "voter_replay_skip_budget".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Persist the artifacts produced when view 1 is skipped. Replaying the // nullification enters view 2 without advancing the finalized floor. let skipped_view = View::new(1); let skipped_round = Round::new(epoch, skipped_view); let (mut nullifies, nullification) = build_nullification(&schemes, skipped_round, quorum); let local_nullify = nullifies.remove(0); seed_voter_journal_artifacts( context.child("seed_journal"), partition.clone(), page_cache.clone(), &schemes[0], skipped_view, vec![ Artifact::Nullify(local_nullify), Artifact::Nullification(nullification), ], ) .await; let (mut mailbox, mut batcher_receiver, _resolver_receiver, _relay, _reporter, _handle) = start_voter_with_floor( &mut context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterFloorStart { partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), skip_budget: 1, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; let recovered_view = View::new(2); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, finalized, .. } => { assert_eq!(current, recovered_view); assert_eq!(finalized, View::zero()); break; } batcher::Message::Constructed(_) => {} } } // The replayed term has exhausted budget 1, so this one-shot hint must remain pending // rather than trigger a skip before the five-second leader deadline. mailbox.timeout( Round::new(epoch, recovered_view), TimeoutReason::LeaderNullify, ); let no_skip_deadline = context.current() + Duration::from_secs(1); loop { select! { _ = context.sleep_until(no_skip_deadline) => break, msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Nullify(nullify)) = msg.unwrap() && nullify.view() == recovered_view { panic!("replay restored skip budget for {recovered_view}"); } }, } } // Finalizing view 1 moves the first unfinalized term to view 2. The // retained hint is now eligible and must fire before the ordinary deadline. let proposal = Proposal::new( skipped_round, View::zero(), Sha256::hash(&[b"restore-skip-budget"]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); let restored_deadline = context.current() + Duration::from_secs(1); loop { select! { _ = context.sleep_until(restored_deadline) => { panic!("finalization did not restore skip budget for {recovered_view}"); }, msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Nullify(nullify)) = msg.unwrap() && nullify.view() == recovered_view { break; } }, } } }); } #[test_traced] fn test_nullification_notification_omits_floor_but_retry_broadcasts_entry() { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"nullification_floor_broadcast", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let observer = participants[1].clone(); let (_, mut certificates) = oracle .control(observer.clone()) .register(1, TEST_QUOTA) .await .unwrap(); oracle .add_link( me, observer, Link { latency: Duration::ZERO, jitter: Duration::ZERO, success_rate: probability!(1.0), }, ) .await .unwrap(); let (mut mailbox, mut batcher, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterOptions { leader_timeout: Duration::from_secs(5), timeout_retry: Duration::from_millis(100), ..Default::default() }, ) .await; assert!(matches!( batcher.recv().await.unwrap(), batcher::Message::Update { current, .. } if current == View::new(1) )); advance_to_view(&mut mailbox, &mut batcher, &schemes, quorum, View::new(2)).await; loop { match batcher.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == View::new(2) => { break; } _ => {} } } let (_, encoded) = certificates.recv().await.unwrap(); let certificate: Certificate = Certificate::decode_cfg(encoded, &schemes[0].certificate_codec_config()).unwrap(); assert!(matches!( certificate, Certificate::Finalization(finalization) if finalization.view() == View::new(1) )); let (_, nullification) = build_nullification(&schemes, Round::new(epoch, View::new(2)), quorum); mailbox.recovered(Certificate::Nullification(nullification.clone())); let (_, encoded) = certificates.recv().await.unwrap(); let certificate: Certificate = Certificate::decode_cfg(encoded, &schemes[0].certificate_codec_config()).unwrap(); assert!(matches!( certificate, Certificate::Nullification(received) if received == nullification )); select! { message = certificates.recv() => { panic!("unexpected certificate after nullification: {message:?}"); }, _ = context.sleep(Duration::from_millis(10)) => {}, } loop { match batcher.recv().await.unwrap() { batcher::Message::Update { current, .. } if current == View::new(3) => break, _ => {} } } mailbox.timeout(Round::new(epoch, View::new(3)), TimeoutReason::Inactivity); loop { match batcher.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(3) => { break; } _ => {} } } let (_, encoded) = certificates.recv().await.unwrap(); let certificate: Certificate = Certificate::decode_cfg(encoded, &schemes[0].certificate_codec_config()).unwrap(); assert!(matches!( certificate, Certificate::Nullification(received) if received == nullification )); }); } fn stall_timeout_nullifies_current_view(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"stall_timeout_nullifies_current_view".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let (network, oracle) = Network::new( context.child("network"), NConfig { max_size: 1024 * 1024, max_peers_per_set: NZUsize!(1), disconnect_on_block: true, tracked_peer_sets: NZUsize!(1), }, ); network.start(); let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(2)), Duration::from_secs(5), ViewDelta::new(0), ); let first_round = Round::new(Epoch::new(333), View::new(1)); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let leader_idx = built_elector.elect(first_round, ()); let leader = participants[usize::from(leader_idx)].clone(); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(3), certification_timeout: Duration::from_secs(4), timeout_retry: Duration::from_secs(2), certify_latency_ms: 3500.0, ..Default::default() }, ) .await; match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => {} _ => panic!("expected initial update"), } let view_1 = View::new(1); let genesis = Sha256::hash(&[&(bytes::Bytes::from_static(b"genesis"), Epoch::new(333)).encode()]); let proposal_1 = Proposal::new( Round::new(Epoch::new(333), view_1), View::zero(), Sha256::hash(&[b"same_term_timeout_view_1"]), ); let contents = (proposal_1.round, genesis, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_1.payload, contents)); mailbox.proposal(proposal_1.clone()); let (_, notarization_1) = build_notarization(&schemes, &proposal_1, quorum); mailbox.recovered(Certificate::Notarization(notarization_1)); loop { if let batcher::Message::Update { current, .. } = batcher_receiver.recv().await.unwrap() && current == View::new(2) { break; } } select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) => { assert_eq!(nullify.view(), View::new(2)); } batcher::Message::Update { .. } => { panic!("expected current-view nullify before the next update"); } batcher::Message::Constructed(Vote::Notarize(_)) => { panic!("expected same-term timeout nullify, got notarize"); } batcher::Message::Constructed(Vote::Finalize(_)) => { panic!("expected same-term timeout nullify, got finalize"); } }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected current-view nullify before leader timeout expired"); } } }); } #[test_traced] fn test_stall_timeout_nullifies_current_view() { stall_timeout_nullifies_current_view::<_, _>(bls12381_threshold_vrf::fixture::); stall_timeout_nullifies_current_view::<_, _>(bls12381_threshold_vrf::fixture::); stall_timeout_nullifies_current_view::<_, _>(bls12381_multisig::fixture::); stall_timeout_nullifies_current_view::<_, _>(bls12381_multisig::fixture::); stall_timeout_nullifies_current_view::<_, _>(ed25519::fixture); stall_timeout_nullifies_current_view::<_, _>(secp256r1::fixture); } fn finalize_resumes_after_same_term_heal(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"finalize_resumes_after_same_term_heal".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let (network, oracle) = Network::new( context.child("network"), NConfig { max_size: 1024 * 1024, max_peers_per_set: NZUsize!(1), disconnect_on_block: true, tracked_peer_sets: NZUsize!(1), }, ); network.start(); let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let elector = RoundRobin::::default().with_term(TermLength::new(NZU32!(3)), Duration::from_secs(20), ViewDelta::new(0)); let (mut mailbox, mut batcher_receiver, _, _relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(2), timeout_retry: Duration::from_secs(10), certify_latency_ms: 0.0, ..Default::default() }, ) .await; match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => {} _ => panic!("expected initial update"), } // The leader of term [1,3] never proposes, so the leader timeout // fires and we nullify view 1. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) => { assert_eq!(nullify.view(), View::new(1)); break; } batcher::Message::Update { .. } => {} _ => panic!("expected nullify for view 1"), } } // Views 1 and 2 notarize and certify anyway, advancing us to view // 3. The finalize votes are blocked by the same-term nullify at // view 1 (no finalization observed yet). let view_1 = View::new(1); let proposal_1 = Proposal::new( Round::new(Epoch::new(333), view_1), View::zero(), Sha256::hash(&[b"finalize_resume_view_1"]), ); let (_, notarization_1) = build_notarization(&schemes, &proposal_1, quorum); mailbox.recovered(Certificate::Notarization(notarization_1)); let view_2 = View::new(2); let proposal_2 = Proposal::new( Round::new(Epoch::new(333), view_2), view_1, Sha256::hash(&[b"finalize_resume_view_2"]), ); let (_, notarization_2) = build_notarization(&schemes, &proposal_2, quorum); mailbox.recovered(Certificate::Notarization(notarization_2)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, .. } if current == View::new(3) => break, batcher::Message::Constructed(Vote::Finalize(_)) => { panic!("finalize must be blocked by the same-term nullify"); } _ => {} } } // The finalization for view 1 arrives: our nullify can never form // a nullification, so the gate heals. let (_, finalization_1) = build_finalization(&schemes, &proposal_1, quorum); mailbox.recovered(Certificate::Finalization(finalization_1)); // View 3 (same term) notarizes and certifies: with the gate // healed, its finalize vote must be broadcast. let view_3 = View::new(3); let proposal_3 = Proposal::new( Round::new(Epoch::new(333), view_3), view_2, Sha256::hash(&[b"finalize_resume_view_3"]), ); let (_, notarization_3) = build_notarization(&schemes, &proposal_3, quorum); mailbox.recovered(Certificate::Notarization(notarization_3)); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Finalize(finalize)) = msg.unwrap() { assert_eq!( finalize.view(), View::new(3), "finalize voting should resume at the first view certified after the heal" ); break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected finalize for view 3 after heal"); } } } }); } #[test_traced] fn test_finalize_resumes_after_same_term_heal() { finalize_resumes_after_same_term_heal::<_, _>(bls12381_threshold_vrf::fixture::); finalize_resumes_after_same_term_heal::<_, _>(bls12381_threshold_vrf::fixture::); finalize_resumes_after_same_term_heal::<_, _>(bls12381_multisig::fixture::); finalize_resumes_after_same_term_heal::<_, _>(bls12381_multisig::fixture::); finalize_resumes_after_same_term_heal::<_, _>(ed25519::fixture); finalize_resumes_after_same_term_heal::<_, _>(secp256r1::fixture); } /// Regression: an optimistic future proposal request must survive the /// loop iteration that handles the current view's timeout while the /// request is in flight. #[test_traced] fn test_optimistic_future_proposal_survives_interleaved_timeout() { let n = 5; let namespace = b"optimistic_future_proposal_survives_interleaved_timeout".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(5)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers( context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(2), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let (_mailbox, mut batcher_receiver, _, _relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(2), // The certification timeout fires between the view-2 // optimistic proposal request (issued with view 1's // notarize after one propose latency) and its response // (a second propose latency later): handling the view-1 // timeout must not drop the in-flight request. certification_timeout: Duration::from_millis(300), timeout_retry: Duration::from_secs(10), local_index, propose_latency_ms: 200.0, ..Default::default() }, ) .await; let mut saw_view_1_notarize = false; let mut saw_view_1_nullify = false; loop { select! { msg = batcher_receiver.recv() => { match msg.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Notarize(notarize)) => { if notarize.view() == View::new(1) { saw_view_1_notarize = true; } if notarize.view() == View::new(2) { assert!( saw_view_1_notarize, "expected view 1 notarize before optimistic view 2 notarize" ); assert!( saw_view_1_nullify, "expected the view-1 timeout to interleave before the view-2 notarize" ); break; } } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(1) => { saw_view_1_nullify = true; } _ => {} } }, _ = context.sleep(Duration::from_secs(8)) => { panic!( "expected optimistic notarize for view 2 despite interleaved timeout handling" ); } } } }); } /// The next view's proposal request reaches the application while the /// current view's notarize is waiting for journal sync. #[test_traced] fn test_next_propose_dispatches_before_sync() { let n = 5; let namespace = b"next_propose_dispatches_before_sync".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(5)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers( context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(2), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let pending_syncs = PendingSyncs::default(); let voter_context = DelayedSyncContext { inner: context.child("voter"), pending: pending_syncs.clone(), }; let propose_requests = Arc::new(Mutex::new(Vec::new())); let (_mailbox, mut batcher_receiver, _, _relay, _) = setup_voter( &voter_context, &oracle, &participants, &schemes, elector, VoterOptions { local_index, propose_latency_ms: 200.0, propose_requests: Some(propose_requests.clone()), ..Default::default() }, ) .await; // Once the view 1 propose request is in flight, gate the journal so // the sync covering the view 1 notarize blocks until released. wait_for_request(&context, &propose_requests, View::new(1), Viewable::view).await; pending_syncs.arm(); // The voter appends the view 1 notarize and blocks on the gated // sync. The view 2 propose request must still reach the application. let deferred = next_pending_sync(&pending_syncs); deferred .blocked .await .expect("view 1 journal sync was dropped"); wait_for_request(&context, &propose_requests, View::new(2), Viewable::view).await; // Nothing constructed for view 1 may reach the batcher while its // sync is blocked. while let Some(msg) = batcher_receiver.recv().now_or_never().flatten() { assert!( !matches!( msg, batcher::Message::Constructed(Vote::Notarize(ref notarize)) if notarize.view() == View::new(1) ), "view 1 notarize reached the batcher before journal sync" ); } assert_eq!( pending_syncs.calls(), 1, "expected one journal sync after the gate was armed" ); // Release the sync. The view 1 notarize must now be published. deferred.release.send(Ok(())).unwrap(); pending_syncs.unblock(); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = msg.unwrap() && notarize.view() == View::new(1) { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected notarize for view 1 after journal sync"); } } } }); } /// An exited view's pending proposal must not suppress replacement work. #[test_collect_traces] fn test_stale_propose_does_not_block_replacement_dispatch(traces: TraceStorage) { let n = 1; let namespace = b"stale_propose_does_not_block_replacement_dispatch".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(2)), Duration::from_secs(30), ViewDelta::new(2), ); let propose_requests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher_receiver, _, _relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_millis(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), propose_requests: Some(propose_requests.clone()), stall_proposals: true, ..Default::default() }, ) .await; assert!(matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Update { .. } )); // The view 1 propose request stalls forever, occupying the pending // request slot. wait_for_request(&context, &propose_requests, View::new(1), Viewable::view).await; // The leader timeout fires while the request is still outstanding. let nullify = loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Nullify(nullify)) = msg.unwrap() && nullify.view() == View::new(1) { break nullify; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected nullify for view 1"); } } }; let nullification = Nullification::from_nullifies(&schemes[0], non_empty![&nullify], &Sequential) .expect("one signer forms quorum"); // The nullification exits view 1. A mid-term view requires its // immediate parent, so view 2 can never propose. The voter skips to // view 3, the term start, which builds on genesis. The stalled // view 1 request must not suppress the view 3 request. mailbox.recovered(Certificate::Nullification(nullification)); wait_for_request(&context, &propose_requests, View::new(3), Viewable::view).await; // The replacement must dispatch at the same-iteration checkpoint, // before the nullification broadcast. let events = traces.get_by_level(Level::DEBUG); let replacement = trace_position( &events, "requested proposal from automaton", "simplex.voter.propose", View::new(3), ); let broadcast = trace_position( &events, "broadcasting nullification", "simplex.voter.notify", View::new(1), ); assert!( replacement < broadcast, "replacement proposal must dispatch before the view 1 nullification broadcast \ (replacement at {replacement}, broadcast at {broadcast})" ); }); } /// A buffered optimistic child starts verification as soon as the local /// parent notarize makes its ancestry available. #[test_collect_traces] fn test_optimistic_future_verify_uses_local_parent_notarize(traces: TraceStorage) { let n = 5; let namespace = b"optimistic_future_verify_uses_local_parent_notarize".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(5)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(2), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let leader_idx = built_elector.elect(Round::new(epoch, View::new(1)), ()); let local_index = (usize::from(leader_idx) + 1) % participants.len(); let leader = participants[usize::from(leader_idx)].clone(); let verify_requests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { // Timeouts long enough that none fires within the test: // any progress past view 1 can only come from optimistic // verification against the local parent notarize. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), local_index, verify_latency_ms: 200.0, verify_requests: Some(verify_requests.clone()), ..Default::default() }, ) .await; match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => {} _ => panic!("expected initial update"), } let genesis = mocks::application::genesis::(epoch); let proposal_1 = Proposal::new( Round::new(epoch, View::new(1)), View::zero(), Sha256::hash(&[b"optimistic_future_verify_view_1"]), ); let contents_1 = (proposal_1.round, genesis, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_1.payload, contents_1)); mailbox.proposal(proposal_1.clone()); wait_for_request(&context, &verify_requests, View::new(1), |view| *view).await; let proposal_2 = Proposal::new( Round::new(epoch, View::new(2)), View::new(1), Sha256::hash(&[b"optimistic_future_verify_view_2"]), ); let contents_2 = (proposal_2.round, proposal_1.payload, 1u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_2.payload, contents_2)); mailbox.proposal(proposal_2.clone()); loop { select! { msg = batcher_receiver.recv() => { match msg.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == View::new(2) => { break; } _ => {} } }, _ = context.sleep(Duration::from_secs(8)) => { panic!( "expected optimistic notarize for view 2 after local parent notarize" ); } } } // Verification for view 2 must dispatch at the same-iteration // checkpoint, before the view 1 notarize broadcast. let events = traces.get_by_level(Level::DEBUG); let verify_next = trace_position( &events, "requested proposal verification", "simplex.voter.verify", View::new(2), ); let broadcast = trace_position( &events, "broadcasting notarize", "simplex.voter.notify", View::new(1), ); assert!( verify_next < broadcast, "view 2 verification must dispatch before the view 1 notarize broadcast \ (verify at {verify_next}, broadcast at {broadcast})" ); }); } /// A pipelined handoff proposes and casts its notarize vote across the term boundary /// before any certificate forms. Voting for the outgoing term's final view is enough for /// the incoming leader to issue its proposal. #[test_traced] fn test_pipelined_handoff_proposes_across_term_boundary() { let n = 5; let namespace = b"pipelined_handoff_proposes_across_term_boundary".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(2)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(1), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); // Views 1 and 2 form term 1. View 3 starts term 2. let outgoing_idx = built_elector.elect(Round::new(epoch, View::new(1)), ()); let outgoing = participants[usize::from(outgoing_idx)].clone(); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(3)), ())); assert_ne!(usize::from(outgoing_idx), local_index); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { // Timeouts long enough that none fires within the test: // no certificate is ever delivered, so a view-3 notarize // can only come from a pipelined handoff. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), local_index, propose_latency_ms: 10.0, handoff: Handoff::Vote(()), ..Default::default() }, ) .await; let genesis = mocks::application::genesis::(epoch); // Feed the outgoing leader's proposals for both views of term 1. let proposal_1 = Proposal::new( Round::new(epoch, View::new(1)), View::zero(), Sha256::hash(&[b"pipelined_handoff_view_1"]), ); let contents_1 = (proposal_1.round, genesis, 0u64).encode(); relay.broadcast(&outgoing, Recipients::All, (proposal_1.payload, contents_1)); mailbox.proposal(proposal_1.clone()); let proposal_2 = Proposal::new( Round::new(epoch, View::new(2)), View::new(1), Sha256::hash(&[b"pipelined_handoff_view_2"]), ); let contents_2 = (proposal_2.round, proposal_1.payload, 1u64).encode(); relay.broadcast(&outgoing, Recipients::All, (proposal_2.payload, contents_2)); mailbox.proposal(proposal_2.clone()); let mut saw_view_2_notarize = false; loop { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = batcher_receiver.recv().await.unwrap() { if notarize.view() == View::new(2) { saw_view_2_notarize = true; } if notarize.view() == View::new(3) { assert!( saw_view_2_notarize, "expected the outgoing tip's notarize before the \ handoff notarize" ); assert_eq!( notarize.proposal.parent, View::new(2), "handoff proposal must build on the outgoing term's final view" ); break; } } } assert_handoff_metrics( &context.encode(), "actor", &[ ("RelayedBeforeCertification", 1), ("VotedBeforeCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[], ); }); } /// A locally pipelined term-start proposal may notarize before its parent certifies, but /// application certification requests remain parent-first. #[test_traced] fn test_pipelined_handoff_certifies_parent_before_child() { let n = 1; let namespace = b"pipelined_handoff_certifies_parent_before_child".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let requests: CertificationRequests = Arc::new(Mutex::new(Vec::new())); let requests_for_app = requests.clone(); let certifier = mocks::application::Certifier::Controlled(Box::new(move |round, _, response| { requests_for_app.lock().push((round.view(), response)); })); let elector = RoundRobin::::default(); let (mut mailbox, mut batcher_receiver, _, _, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), certifier, handoff: Handoff::Vote(()), ..Default::default() }, ) .await; // A singleton's local votes notarize both views while the parent // application request remains under test control. let mut parent = None; let mut child = None; while parent.is_none() || child.is_none() { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = batcher_receiver.recv().await.unwrap() { match notarize.view() { view if view == View::new(1) => { parent = Some(notarize.proposal); } view if view == View::new(2) => { child = Some(notarize.proposal); } _ => {} } } } let parent = parent.expect("parent proposal"); let child = child.expect("child proposal"); assert_eq!(child.parent, parent.view()); // Deliver the child certificate first. The mailbox is FIFO, so by // the time the parent's request appears, a child request that did // not wait for the parent would already be queued. let (_, child_notarization) = build_notarization(&schemes, &child, 1); mailbox.recovered(Certificate::Notarization(child_notarization)); let (_, parent_notarization) = build_notarization(&schemes, &parent, 1); mailbox.recovered(Certificate::Notarization(parent_notarization)); let parent_response = take_certification_request(&context, &requests, View::new(1)).await; assert!( !requests .lock() .iter() .any(|(view, _)| *view == View::new(2)), "child certification ran ahead of its parent" ); parent_response .send(true) .expect("parent certification receiver must remain open"); let child_response = take_certification_request(&context, &requests, View::new(2)).await; child_response .send(true) .expect("child certification receiver must remain open"); loop { if matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Constructed(Vote::Finalize(finalize)) if finalize.view() == View::new(2) ) { break; } } assert!(reporter.certifications.lock().contains_key(&View::new(1))); assert!(reporter.certifications.lock().contains_key(&View::new(2))); }); } /// Leader timeout of [`HandoffFixture`] voters and their restarted replacements. Timeout /// assertions wait past it. const HANDOFF_LEADER_TIMEOUT: Duration = Duration::from_secs(10); /// Certification timeout of [`HandoffFixture`] voters and their restarted replacements. const HANDOFF_CERTIFICATION_TIMEOUT: Duration = Duration::from_secs(10); /// Timeout retry interval of [`HandoffFixture`] voters and their restarted replacements. const HANDOFF_RETRY_TIMEOUT: Duration = Duration::from_secs(30); /// Time for the voter to drain its mailbox, including responses that arrive after the mock /// application's 1ms latencies, when no observable event marks that it has done so. const SETTLE: Duration = Duration::from_millis(50); /// Metrics prefix of the fixture voter: the `delayed` sync context plus /// the `actor` child that `setup_voter` creates. const HANDOFF_ACTOR_METRICS: &str = "delayed_actor"; /// A five-validator voter that leads view 3, the first view of the second two-view term, /// after voting for the uncertified outgoing tip at view 2. The application holds the /// certification of view 2 for the test. /// /// `C` is the handoff [`Candidate`] the test answers, or `()` when the test does not take /// the application's handoff response. struct HandoffFixture { participants: Vec, schemes: Vec, oracle: Oracle, elector: RoundRobin, local_index: usize, parent: Proposal, certification_requests: CertificationRequests, pending_syncs: PendingSyncs, actor_handle: Arc>>>, mailbox: Mailbox, batcher: mailbox::Receiver>, resolver: mailbox::Receiver>, relay: Arc>, propose_requests: ProposeRequests, handoff_responses: HandoffResponses, candidate: C, } /// The handoff candidate the mock application built for view 3, with its held response. struct Candidate { /// The candidate the mock built, forwarded unchanged by `respond`. proposal: Handoff, /// The candidate's payload, for observing relays and votes. digest: Sha256Digest, response: Option>>, } impl HandoffFixture<()> { /// Starts the fixture voter with `options` and drives it to the uncertified outgoing /// tip. The test does not take the application's handoff response. async fn start(context: &mut deterministic::Context, options: VoterOptions) -> Self { let n = 5; let epoch = Epoch::new(333); let Fixture { participants, schemes, .. } = ed25519::fixture(context, b"pipelined_handoff_linear", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(2)), Duration::from_secs(30), ViewDelta::new(1), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(3)), ())); let outgoing_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let outgoing = participants[outgoing_index].clone(); let propose_requests = Arc::new(Mutex::new(Vec::new())); let handoff_responses = Arc::new(Mutex::new(Vec::new())); let certification_requests = Arc::new(Mutex::new(Vec::new())); let actor_handle = Arc::new(Mutex::new(None)); let pending_syncs = PendingSyncs::default(); let voter_context = DelayedSyncContext { inner: context.child("delayed"), pending: pending_syncs.clone(), }; let (mut mailbox, mut batcher, mut resolver, relay, _) = setup_voter( &voter_context, &oracle, &participants, &schemes, elector.clone(), VoterOptions { leader_timeout: HANDOFF_LEADER_TIMEOUT, certification_timeout: HANDOFF_CERTIFICATION_TIMEOUT, timeout_retry: HANDOFF_RETRY_TIMEOUT, local_index, actor_handle: Some(actor_handle.clone()), certifier: hold(View::new(2), certification_requests.clone()), propose_requests: Some(propose_requests.clone()), handoff_responses: Some(handoff_responses.clone()), ..options }, ) .await; let parent = prepare_pipelined_handoff_tip( &schemes, local_index, &outgoing, &mut mailbox, &mut batcher, &mut resolver, &relay, ) .await; Self { participants, schemes, oracle, elector, local_index, parent, certification_requests, pending_syncs, actor_handle, mailbox, batcher, resolver, relay, propose_requests, handoff_responses, candidate: (), } } } impl HandoffFixture { /// Starts the fixture voter with `decision` and takes the candidate the application /// returns for the view 3 handoff request, leaving the test to answer it. async fn new(context: &mut deterministic::Context, decision: Handoff<()>) -> Self { let HandoffFixture { participants, schemes, oracle, elector, local_index, parent, certification_requests, pending_syncs, actor_handle, mailbox, batcher, resolver, relay, propose_requests, handoff_responses, candidate: (), } = HandoffFixture::start( context, VoterOptions { handoff: decision, ..Default::default() }, ) .await; let (proposal, response) = take_proposal_response(context, &handoff_responses).await; let (Handoff::Vote(digest) | Handoff::Stage(digest)) = proposal else { panic!("mock must return a candidate"); }; assert_eq!( matches!(proposal, Handoff::Vote(_)), decision == Handoff::Vote(()) ); Self { participants, schemes, oracle, elector, local_index, parent, certification_requests, pending_syncs, actor_handle, mailbox, batcher, resolver, relay, propose_requests, handoff_responses, candidate: Candidate { proposal, digest, response: Some(response), }, } } /// Answers the pending handoff with the built candidate. fn respond(&mut self) { self.candidate .response .take() .expect("handoff response must be pending") .send(self.candidate.proposal) .expect("handoff request must remain open"); } /// Answers the pending handoff with [`Handoff::Wait`]. fn respond_wait(&mut self) { self.candidate .response .take() .expect("handoff response must be pending") .send(Handoff::Wait) .expect("handoff request must remain open"); } } impl HandoffFixture { fn quorum(&self) -> u32 { quorum(u32::try_from(self.schemes.len()).expect("test fixture must fit u32")) } /// Counts proposal requests the application received for `view`, /// including the handoff request. fn requests_for(&self, view: View) -> usize { self.propose_requests .lock() .iter() .filter(|request| request.view() == view) .count() } /// Waits for exactly one ordinary replacement with the full captured context. async fn wait_for_replacement(&self, context: &deterministic::Context) { while self.requests_for(View::new(3)) < 2 { context.sleep(Duration::from_millis(1)).await; } let contexts: Vec<_> = self .propose_requests .lock() .iter() .filter(|request| request.view() == View::new(3)) .cloned() .collect(); assert_eq!(contexts.len(), 2); assert_eq!( contexts[0], contexts[1], "ordinary request must reuse the captured context" ); assert_eq!(contexts[1].parent, (View::new(2), self.parent.payload)); } /// Delivers the captured parent's notarization and returns its /// certification request. async fn certify_parent( &mut self, context: &deterministic::Context, ) -> oneshot::Sender { let (_, notarization) = build_notarization(&self.schemes, &self.parent, self.quorum()); self.mailbox .recovered(Certificate::Notarization(notarization)); take_certification_request(context, &self.certification_requests, self.parent.view()) .await } /// Answers the parent's certification request and returns the handle /// that releases its blocked journal sync. async fn block_certification( &mut self, certified: oneshot::Sender, ) -> oneshot::Sender> { self.pending_syncs.arm(); certified.send(true).unwrap(); let journal = next_pending_sync(&self.pending_syncs); journal .blocked .await .expect("certification journal sync started"); journal.release } /// Releases the blocked sync and waits until the voter has processed /// the parent's certification. async fn release_certification(&mut self, release: oneshot::Sender>) { release.send(Ok(())).unwrap(); self.pending_syncs.unblock(); loop { if matches!(self.resolver.recv().await.unwrap(), MailboxMessage::Certified { notarization, success: true, .. } if notarization.view() == View::new(2)) { return; } } } /// Answers the parent's certification request and waits until the /// voter has processed it. async fn finish_certification(&mut self, certified: oneshot::Sender) { let release = self.block_certification(certified).await; self.release_certification(release).await; } /// Registers the next participant as a relay observer. fn observer(&self) -> mpsc::UnboundedReceiver<(Sha256Digest, Bytes)> { self.observer_of(&self.relay) } /// Registers the next participant as an observer of `relay`, such as the relay of a /// restarted voter. fn observer_of( &self, relay: &mocks::relay::Relay, ) -> mpsc::UnboundedReceiver<(Sha256Digest, Bytes)> { let observer = self.participants[(self.local_index + 1) % self.participants.len()].clone(); relay.register(observer) } /// Aborts the fixture voter and starts its replacement on a `restarted` context, with /// the fixture's participants, elector, and timeouts and the remaining `options`. async fn restart( &mut self, context: &deterministic::Context, options: VoterOptions, ) -> ( Mailbox, mailbox::Receiver>, Arc>, ) { let handle = self.actor_handle.lock().take().unwrap(); handle.abort(); assert!(handle.await.is_err()); let restarted_context = context.child("restarted"); let (mailbox, batcher, _, relay, _) = setup_voter( &restarted_context, &self.oracle, &self.participants, &self.schemes, self.elector.clone(), VoterOptions { local_index: self.local_index, leader_timeout: HANDOFF_LEADER_TIMEOUT, certification_timeout: HANDOFF_CERTIFICATION_TIMEOUT, timeout_retry: HANDOFF_RETRY_TIMEOUT, ..options }, ) .await; (mailbox, batcher, relay) } } /// Waits for a `handoff_events` label to count an event. async fn wait_for_handoff_event(context: &deterministic::Context, event: &str) { wait_for_handoff_metric(context, "handoff_events", "event", event).await; } /// Waits for a `handoff_abandoned` reason to count an abandonment. async fn wait_for_handoff_abandoned(context: &deterministic::Context, reason: &str) { wait_for_handoff_metric(context, "handoff_abandoned", "reason", reason).await; } /// Waits until the voter constructs a nullify vote for `view`. async fn wait_for_local_nullify( batcher: &mut mailbox::Receiver>, view: View, ) { loop { if let batcher::Message::Constructed(Vote::Nullify(nullify)) = batcher.recv().await.expect("batcher must remain open") && nullify.view() == view { return; } } } /// Waits for a handoff metric label to count at least one event. async fn wait_for_handoff_metric( context: &deterministic::Context, family: &str, label: &str, value: &str, ) { loop { if labeled_metric_snapshot(&context.encode(), HANDOFF_ACTOR_METRICS, family, label) .get(value) .is_some_and(|count| *count > 0) { return; } context.sleep(Duration::from_millis(1)).await; } } /// Asserts that a batcher `message` is not a notarize vote for any `stale` candidate. fn assert_no_stale_vote( message: &batcher::Message, stale: &[Sha256Digest], ) { assert!( !matches!(message, batcher::Message::Constructed(Vote::Notarize(vote)) if stale.contains(&vote.proposal.payload)), "stale handoff must not vote" ); } /// Drains the relay observer and the batcher, asserting that neither relays nor votes for /// any `stale` candidate. fn assert_handoff_silent( relayed: &mut mpsc::UnboundedReceiver<(Sha256Digest, Bytes)>, batcher: &mut mailbox::Receiver>, stale: &[Sha256Digest], ) { while let Some((relayed_digest, _)) = relayed.recv().now_or_never().flatten() { assert!( !stale.contains(&relayed_digest), "stale handoff must not relay" ); } while let Some(message) = batcher.recv().now_or_never().flatten() { assert_no_stale_vote(&message, stale); } } /// Waits for the relay of a candidate held for its parent's certification. async fn observe_handoff_relay( relayed: &mut mpsc::UnboundedReceiver<(Sha256Digest, Bytes)>, digest: Sha256Digest, ) { let (relayed_digest, _) = relayed .recv() .await .expect("relay observer must remain open"); assert_eq!(relayed_digest, digest, "held handoff must be relayed"); } /// Waits for the notarize vote that releases a held candidate. The candidate was /// relayed when it was held, so no second relay may arrive. async fn observe_handoff_vote( relayed: &mut mpsc::UnboundedReceiver<(Sha256Digest, Bytes)>, batcher: &mut mailbox::Receiver>, digest: Sha256Digest, ) { loop { select! { message = relayed.recv() => { let (relayed_digest, _) = message.expect("relay observer must remain open"); assert_ne!(relayed_digest, digest, "released handoff must not relay again"); }, message = batcher.recv() => { if let batcher::Message::Constructed(Vote::Notarize(vote)) = message.expect("batcher must remain open") && vote.view() == View::new(3) { assert_eq!(vote.proposal.payload, digest, "released handoff must be voted"); return; } }, } } } async fn observe_handoff_relay_and_vote( relayed: &mut mpsc::UnboundedReceiver<(Sha256Digest, Bytes)>, batcher: &mut mailbox::Receiver>, digest: Sha256Digest, ) { let mut observed_relay = false; let mut observed_vote = false; while !observed_relay || !observed_vote { select! { message = relayed.recv() => { let (relayed_digest, _) = message.expect("relay observer must remain open"); assert_eq!(relayed_digest, digest, "retained handoff must be relayed"); observed_relay = true; }, message = batcher.recv() => { if let batcher::Message::Constructed(Vote::Notarize(vote)) = message.expect("batcher must remain open") && vote.view() == View::new(3) { assert_eq!(vote.proposal.payload, digest, "retained handoff must be voted"); observed_vote = true; } }, } } } #[test_traced] fn test_pipelined_handoff_build_before_certification() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; let certified = fixture.certify_parent(&context).await; let release = fixture.block_certification(certified).await; assert_handoff_silent( &mut relayed, &mut fixture.batcher, &[fixture.candidate.digest], ); fixture.release_certification(release).await; observe_handoff_vote(&mut relayed, &mut fixture.batcher, fixture.candidate.digest) .await; assert_eq!( fixture.requests_for(View::new(3)), 1, "held handoff must not rebuild" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("VotedAfterCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[], ); }); } /// Certification completes before the application answers. Entering the child /// view on the certified parent keeps the pending build, and the single build is /// relayed and voted for once, after certification, for either decision. async fn certification_first_reuses_build( context: &mut deterministic::Context, decision: Handoff<()>, ) { let mut fixture = HandoffFixture::new(context, decision).await; let mut relayed = fixture.observer(); let certified = fixture.certify_parent(context).await; fixture.finish_certification(certified).await; assert!( !fixture .candidate .response .as_ref() .expect("handoff response must be held") .is_closed(), "parent certification must not cancel the pending build" ); fixture.respond(); observe_handoff_relay_and_vote( &mut relayed, &mut fixture.batcher, fixture.candidate.digest, ) .await; assert_eq!(fixture.requests_for(View::new(3)), 1, "must not rebuild"); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedAfterCertification", 1), ("VotedAfterCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[], ); } /// [`certification_first_reuses_build`] under [`Handoff::Stage`]. #[test_traced] fn test_pipelined_handoff_certification_first_reuses_build_stage() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { certification_first_reuses_build(&mut context, Handoff::Stage(())).await; }); } /// [`certification_first_reuses_build`] under [`Handoff::Vote`]. #[test_traced] fn test_pipelined_handoff_certification_first_reuses_build_vote() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { certification_first_reuses_build(&mut context, Handoff::Vote(())).await; }); } /// Parent finalization releases a held build like certification does. #[test_traced] fn test_pipelined_handoff_held_build_released_by_parent_finalization() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; let (_, finalization) = build_finalization( &peers(&fixture.schemes, fixture.local_index), &fixture.parent, fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Finalization(finalization)); observe_handoff_vote(&mut relayed, &mut fixture.batcher, fixture.candidate.digest) .await; assert_eq!( fixture.requests_for(View::new(3)), 1, "held handoff must not rebuild" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("VotedAfterCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[], ); }); } /// An already waiting request starts its ordinary build before the parent /// sync completes, but its response cannot relay or vote for the child yet. #[test_traced] fn test_pipelined_handoff_waiting_build_overlaps_certification_sync() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond_wait(); wait_for_handoff_event(&context, "WaitReturned").await; let certified = fixture.certify_parent(&context).await; let release = fixture.block_certification(certified).await; fixture.wait_for_replacement(&context).await; // The ordinary mock response is available after its 1ms build latency. context.sleep(SETTLE).await; assert_eq!(fixture.requests_for(View::new(3)), 2); assert!( relayed.recv().now_or_never().is_none(), "ordinary build must not relay before sync" ); while let Some(message) = fixture.batcher.recv().now_or_never().flatten() { assert!( !matches!(message, batcher::Message::Constructed(Vote::Notarize(ref vote)) if vote.view() == View::new(3)), "ordinary build must not vote before sync" ); } fixture.release_certification(release).await; let (digest, _) = relayed .recv() .await .expect("ordinary build must relay after sync"); loop { if let batcher::Message::Constructed(Vote::Notarize(vote)) = fixture .batcher .recv() .await .expect("batcher must remain open") && vote.view() == View::new(3) { assert_eq!(vote.proposal.payload, digest); break; } } assert_eq!(fixture.requests_for(View::new(3)), 2); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("WaitReturned", 1), ("Requested", 1)], &[], ); }); } /// A Wait response that arrives while the parent's certification sync is blocked /// is not consumed until the loop resumes. It then dispatches its ordinary /// replacement. #[test_traced] fn test_pipelined_handoff_wait_response_during_certification_sync() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let certified = fixture.certify_parent(&context).await; let release = fixture.block_certification(certified).await; fixture.respond_wait(); context.sleep(SETTLE).await; assert_eq!(fixture.requests_for(View::new(3)), 1); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("Requested", 1)], &[], ); fixture.release_certification(release).await; fixture.wait_for_replacement(&context).await; context.sleep(SETTLE).await; assert_eq!(fixture.requests_for(View::new(3)), 2); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("WaitReturned", 1), ("Requested", 1)], &[], ); }); } /// A late Wait response after durable parent certification needs no further /// parent event to dispatch its ordinary replacement. #[test_traced] fn test_pipelined_handoff_wait_response_after_certification_sync() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let certified = fixture.certify_parent(&context).await; fixture.finish_certification(certified).await; assert_eq!(fixture.requests_for(View::new(3)), 1); fixture.respond_wait(); fixture.wait_for_replacement(&context).await; context.sleep(SETTLE).await; assert_eq!(fixture.requests_for(View::new(3)), 2); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("WaitReturned", 1), ("Requested", 1)], &[], ); }); } /// A waiting handoff request is abandoned when its view exits. #[test_traced] fn test_pipelined_handoff_waiting_request_exits_view() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; fixture.respond_wait(); wait_for_handoff_event(&context, "WaitReturned").await; let certified = fixture.certify_parent(&context).await; let (_, nullification) = build_nullification( &peers(&fixture.schemes, fixture.local_index), Round::new(Epoch::new(333), View::new(3)), fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Nullification(nullification)); loop { if matches!( fixture.batcher.recv().await.unwrap(), batcher::Message::Update { current, .. } if current > View::new(3) ) { break; } } fixture.finish_certification(certified).await; wait_for_handoff_abandoned(&context, "ViewExit").await; assert_eq!( fixture.requests_for(View::new(3)), 1, "no ordinary request follows an exited view" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("WaitReturned", 1), ("Requested", 1)], &[("ViewExit", 1)], ); }); } /// Voting to nullify the parent's view cancels a pending handoff build, since the /// application may verify that parent only after the build completes. The request /// is retained, and an ordinary request for the same context follows if the parent /// certifies after all. #[test_traced] fn test_pipelined_handoff_parent_nullify_cancels_pending_build() { let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let certified = fixture.certify_parent(&context).await; // The parent's certification times out while it and the handoff build are both pending, // so we vote to nullify view 2. wait_for_local_nullify(&mut fixture.batcher, View::new(2)).await; wait_for_handoff_abandoned(&context, "ParentNullify").await; assert!( fixture .candidate .response .as_ref() .expect("handoff response must be held") .is_closed(), "pending handoff build must be cancelled" ); assert_eq!(fixture.requests_for(View::new(3)), 1); // The parent certifies after all, so the retained request becomes an // ordinary request for the captured context. fixture.finish_certification(certified).await; fixture.wait_for_replacement(&context).await; assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("Requested", 1)], &[("ParentNullify", 1)], ); }); } /// A failed certification of the parent invalidates the captured ancestry, so the /// pending handoff build is cancelled before we vote to nullify the parent's view. #[test_traced] fn test_pipelined_handoff_failed_parent_certification_cancels_pending_build() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let certified = fixture.certify_parent(&context).await; certified.send(false).unwrap(); wait_for_handoff_abandoned(&context, "AncestryInvalidated").await; assert!( fixture .candidate .response .as_ref() .expect("handoff response must be held") .is_closed(), "pending handoff build must be cancelled" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("Requested", 1)], &[("AncestryInvalidated", 1)], ); }); } /// A nullification in the parent's term that arrives after the parent certifies /// leaves the pending build in place, since the certified parent stays usable. #[test_traced] fn test_pipelined_handoff_late_parent_nullification_keeps_build() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); let certified = fixture.certify_parent(&context).await; fixture.finish_certification(certified).await; let (votes, nullification) = build_nullification( &peers(&fixture.schemes, fixture.local_index), Round::new(Epoch::new(333), View::new(2)), fixture.quorum(), ); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(fixture.local_index)), "nullification quorum includes the local voter" ); fixture .mailbox .recovered(Certificate::Nullification(nullification)); // Give the voter time to record the nullification before the build // answers. `respond` fails if the voter cancelled the build. context.sleep(SETTLE).await; fixture.respond(); observe_handoff_relay_and_vote( &mut relayed, &mut fixture.batcher, fixture.candidate.digest, ) .await; assert_eq!(fixture.requests_for(View::new(3)), 1, "must not rebuild"); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedAfterCertification", 1), ("VotedAfterCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[], ); }); } /// Replay restores our nullify vote for the uncertified parent's view, so the /// restarted voter does not request a fresh handoff build on that parent. Parent /// certification then yields one ordinary request. #[test_traced] fn test_pipelined_handoff_not_reissued_after_parent_nullify_on_restart() { let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; wait_for_local_nullify(&mut fixture.batcher, View::new(2)).await; let restarted_requests: ProposeRequests = Arc::new(Mutex::new(Vec::new())); let (mut restarted, mut restarted_batcher, _) = fixture .restart( &context, VoterOptions { propose_requests: Some(restarted_requests.clone()), ..Default::default() }, ) .await; // No handoff request follows replay. let deadline = context.current() + Duration::from_secs(1); loop { select! { _ = restarted_batcher.recv() => {}, _ = context.sleep_until(deadline) => break, } } assert!( restarted_requests.lock().is_empty(), "handoff must not be requested on a parent we voted to nullify" ); // Certifying the parent makes the view an ordinary candidate. let (_, notarization) = build_notarization(&fixture.schemes, &fixture.parent, fixture.quorum()); restarted.recovered(Certificate::Notarization(notarization)); while restarted_requests.lock().is_empty() { select! { _ = restarted_batcher.recv() => {}, _ = context.sleep(Duration::from_millis(1)) => {}, } } assert_eq!( request_parents(&restarted_requests), [(View::new(3), View::new(2))] ); assert_handoff_metrics(&context.encode(), "restarted_actor", &[], &[]); }); } #[test_traced] fn test_pipelined_handoff_held_build_rejects_conflicting_parent() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; // The conflicting notarization must exclude the local voter, which already // voted for the original parent. let mut conflicting_parent = fixture.parent.clone(); conflicting_parent.payload = Sha256::hash(&[b"conflicting parent"]); let (votes, notarization) = build_notarization( &peers(&fixture.schemes, fixture.local_index), &conflicting_parent, fixture.quorum(), ); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(fixture.local_index)), "conflicting quorum includes the local voter" ); fixture .mailbox .recovered(Certificate::Notarization(notarization)); let certified = take_certification_request( &context, &fixture.certification_requests, conflicting_parent.view(), ) .await; fixture.finish_certification(certified).await; wait_for_handoff_abandoned(&context, "AncestrySuperseded").await; assert_handoff_silent( &mut relayed, &mut fixture.batcher, &[fixture.candidate.digest], ); // The conflicting notarization supersedes the held build and issues // a second handoff request on the new tip. assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("CandidateReturned", 1), ("Requested", 2), ], &[("AncestrySuperseded", 1)], ); }); } /// Pins the request contract for one view: at most two prepare requests and one ordinary /// proposal request. /// /// The second prepare follows only the notarization of an equivocating outgoing leader's /// other block at the parent view, which replaces the parent this node voted for. The /// ordinary request follows the replacement's certification after the second prepare /// answers `Wait`. Nothing follows the ordinary request. #[test_traced] fn test_pipelined_handoff_request_contract() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; fixture.respond(); wait_for_handoff_event(&context, "RelayedBeforeCertification").await; // The conflicting notarization excludes the local voter, which already voted for // the original parent. let mut conflicting_parent = fixture.parent.clone(); conflicting_parent.payload = Sha256::hash(&[b"conflicting parent"]); let (_, notarization) = build_notarization( &peers(&fixture.schemes, fixture.local_index), &conflicting_parent, fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Notarization(notarization)); let certified = take_certification_request( &context, &fixture.certification_requests, conflicting_parent.view(), ) .await; wait_for_handoff_abandoned(&context, "AncestrySuperseded").await; // The second prepare names the replacement parent. Decline it. let (_, response) = take_proposal_response(&context, &fixture.handoff_responses).await; response .send(Handoff::Wait) .expect("second prepare must remain open"); wait_for_handoff_event(&context, "WaitReturned").await; // Certifying the replacement turns the waiting request into one ordinary proposal. fixture.finish_certification(certified).await; while fixture.requests_for(View::new(3)) < 3 { context.sleep(Duration::from_millis(1)).await; } context.sleep(SETTLE).await; let contexts: Vec<_> = fixture .propose_requests .lock() .iter() .filter(|request| request.view() == View::new(3)) .cloned() .collect(); assert_eq!(contexts.len(), 3, "two prepares and one ordinary proposal"); assert_eq!(contexts[0].parent, (View::new(2), fixture.parent.payload)); assert_eq!( contexts[1].parent, (View::new(2), conflicting_parent.payload) ); assert_eq!( contexts[2].parent, (View::new(2), conflicting_parent.payload) ); assert!( fixture.handoff_responses.lock().is_empty(), "the ordinary request must not reach the prepare path" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("Requested", 2), ("CandidateReturned", 1), ("RelayedBeforeCertification", 1), ("WaitReturned", 1), ], &[("AncestrySuperseded", 1)], ); }); } #[test_traced] fn test_pipelined_handoff_held_build_rejects_parent_nullification() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; let certified = fixture.certify_parent(&context).await; let (_, nullification) = build_nullification( &peers(&fixture.schemes, fixture.local_index), Round::new(Epoch::new(333), View::new(2)), fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Nullification(nullification)); loop { if matches!( fixture.batcher.recv().await.unwrap(), batcher::Message::Update { current, .. } if current == View::new(3) ) { break; } } fixture.finish_certification(certified).await; wait_for_handoff_abandoned(&context, "AncestrySuperseded").await; assert_handoff_silent( &mut relayed, &mut fixture.batcher, &[fixture.candidate.digest], ); // The nullified tip leaves certified ancestry, so the replacement // is an ordinary request rather than a second handoff. while fixture.requests_for(View::new(3)) < 2 { context.sleep(Duration::from_millis(1)).await; } assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[("AncestrySuperseded", 1)], ); }); } #[test_traced] fn test_pipelined_handoff_held_build_exits_view() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; let certified = fixture.certify_parent(&context).await; let (_, nullification) = build_nullification( &peers(&fixture.schemes, fixture.local_index), Round::new(Epoch::new(333), View::new(3)), fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Nullification(nullification)); loop { if matches!( fixture.batcher.recv().await.unwrap(), batcher::Message::Update { current, .. } if current > View::new(3) ) { break; } } fixture.finish_certification(certified).await; wait_for_handoff_abandoned(&context, "ViewExit").await; assert_handoff_silent( &mut relayed, &mut fixture.batcher, &[fixture.candidate.digest], ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[("ViewExit", 1)], ); }); } #[test_traced] fn test_pipelined_handoff_held_build_becomes_ineligible_at_leader_timeout() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); fixture.respond(); observe_handoff_relay(&mut relayed, fixture.candidate.digest).await; let certified = fixture.certify_parent(&context).await; let release = fixture.block_certification(certified).await; // The configured leader timeout expires while the earlier parent // certification journal sync remains blocked. context .sleep(HANDOFF_LEADER_TIMEOUT + Duration::from_secs(1)) .await; fixture.release_certification(release).await; wait_for_handoff_abandoned(&context, "IneligibleAtRecording").await; assert_handoff_silent( &mut relayed, &mut fixture.batcher, &[fixture.candidate.digest], ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[ ("RelayedBeforeCertification", 1), ("CandidateReturned", 1), ("Requested", 1), ], &[("IneligibleAtRecording", 1)], ); }); } /// Restart discards a held candidate: replay re-issues the handoff request instead of /// restoring the candidate, and dropping that request forfeits the view once the parent /// certifies. #[test_traced] fn test_pipelined_handoff_held_build_is_volatile_on_restart() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; fixture.respond(); wait_for_handoff_event(&context, "RelayedBeforeCertification").await; let restarted_responses: HandoffResponses = Arc::new(Mutex::new(Vec::new())); let (mut restarted, mut restarted_batcher, _) = fixture .restart( &context, VoterOptions { handoff: Handoff::Stage(()), handoff_responses: Some(restarted_responses.clone()), ..Default::default() }, ) .await; let (_, fresh) = take_proposal_response(&context, &restarted_responses).await; drop(fresh); let (_, notarization) = build_notarization(&fixture.schemes, &fixture.parent, fixture.quorum()); restarted.recovered(Certificate::Notarization(notarization)); wait_for_local_nullify(&mut restarted_batcher, View::new(3)).await; }); } /// A leader that relays a held handoff candidate and restarts before voting builds a fresh /// candidate, since no journaled vote names the relayed one. Only the rebuilt candidate is /// voted once the parent certifies, and once that vote is journaled another restart does /// not build again. #[test_traced] fn test_pipelined_handoff_restart_before_vote_votes_only_rebuilt_candidate() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // The original voter relays its held candidate without voting for it. let mut fixture = HandoffFixture::new(&mut context, Handoff::Stage(())).await; let mut relayed = fixture.observer(); let relayed_candidate = fixture.candidate.digest; fixture.respond(); observe_handoff_relay(&mut relayed, relayed_candidate).await; assert_handoff_silent(&mut relayed, &mut fixture.batcher, &[relayed_candidate]); // The replacement replays a journal with no vote at view 3 and builds again. It // takes over the fixture's actor handle so the next restart aborts it. let restarted_responses: HandoffResponses = Arc::new(Mutex::new(Vec::new())); let actor_handle = fixture.actor_handle.clone(); let (mut restarted, mut restarted_batcher, restarted_relay) = fixture .restart( &context, VoterOptions { handoff: Handoff::Stage(()), handoff_responses: Some(restarted_responses.clone()), actor_handle: Some(actor_handle.clone()), ..Default::default() }, ) .await; let mut restarted_relayed = fixture.observer_of(&restarted_relay); let (rebuilt_decision, response) = take_proposal_response(&context, &restarted_responses).await; let Handoff::Stage(rebuilt) = rebuilt_decision else { panic!("restarted mock must return a candidate"); }; assert_ne!( rebuilt, relayed_candidate, "restart must build a fresh candidate" ); response .send(rebuilt_decision) .expect("handoff request must remain open"); observe_handoff_relay(&mut restarted_relayed, rebuilt).await; // Certifying the parent releases the rebuilt candidate, and the vote names only it. let (_, notarization) = build_notarization(&fixture.schemes, &fixture.parent, fixture.quorum()); restarted.recovered(Certificate::Notarization(notarization)); observe_handoff_vote(&mut restarted_relayed, &mut restarted_batcher, rebuilt).await; assert_handoff_silent( &mut restarted_relayed, &mut restarted_batcher, &[relayed_candidate], ); // The vote is journaled before it is published, so a further restart replays it // and does not build for view 3 again. let again_requests: ProposeRequests = Arc::new(Mutex::new(Vec::new())); let (_again, mut again_batcher, again_relay) = fixture .restart( &context, VoterOptions { handoff: Handoff::Stage(()), propose_requests: Some(again_requests.clone()), actor_handle: Some(actor_handle), ..Default::default() }, ) .await; let mut again_relayed = fixture.observer_of(&again_relay); // The second replacement nullifies view 3 when its timeout fires, which happens only // after it replayed the journal and passed the point where it would build. Neither // candidate may be relayed or voted before or after that point. let stale = [rebuilt, relayed_candidate]; loop { let message = again_batcher .recv() .await .expect("batcher must remain open"); if matches!(&message, batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(3)) { break; } assert_no_stale_vote(&message, &stale); } assert!( again_requests .lock() .iter() .all(|request| request.view() != View::new(3)), "a journaled vote must latch the view against another build" ); assert_handoff_silent(&mut again_relayed, &mut again_batcher, &stale); }); } /// A dropped handoff response forfeits the term-start view once its parent certifies: the /// voter nullifies the view without requesting an ordinary proposal on that parent. #[test_traced] fn test_pipelined_handoff_dropped_response_nullifies_after_parent_certification() { let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let mut fixture = HandoffFixture::start( &mut context, VoterOptions { drop_proposals: true, handoff: Handoff::Vote(()), ..Default::default() }, ) .await; wait_for_request( &context, &fixture.propose_requests, View::new(3), Viewable::view, ) .await; assert_eq!( request_parents(&fixture.propose_requests), [(View::new(3), View::new(2))] ); let deadline = context.current() + Duration::from_secs(1); let certified = fixture.certify_parent(&context).await; certified.send(true).unwrap(); let mut parent_certified = false; let mut entered_child = false; let mut child_nullified = false; while !parent_certified || !child_nullified { select! { message = fixture.resolver.recv() => { if matches!( message.unwrap(), MailboxMessage::Certified { notarization, success: true, .. } if notarization.view() == View::new(2) ) { parent_certified = true; } }, message = fixture.batcher.recv() => { match message.unwrap() { batcher::Message::Update { current, .. } if current == View::new(3) => { entered_child = true; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(3) => { assert!(entered_child, "child nullified before becoming current"); child_nullified = true; } _ => {} } }, _ = context.sleep_until(deadline) => { panic!( "expected dropped handoff to nullify view 3 before normal deadlines" ); } } } assert_eq!( request_parents(&fixture.propose_requests), [(View::new(3), View::new(2))], "parent certification must not retry a dropped handoff" ); assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("Requested", 1)], &[("ResponseClosed", 1)], ); }); } /// How the application leaves the handoff request unanswered in /// [`reissue_handoff_after_parent_nullification`]. enum Unanswered { /// Every proposal response stays open. Stalled, /// The handoff response closes, and ordinary requests are answered. Closed, } /// Runs a pipelined handoff whose application leaves the request `unanswered`, and then /// nullifies the captured parent while the incoming term-start view is still live. The /// voter must issue a replacement request on the certified fallback before the leader /// timeout. A closed response forfeits only its captured context, so the replacement /// proposes and votes on the fallback. fn reissue_handoff_after_parent_nullification(unanswered: Unanswered) { let closed = matches!(unanswered, Unanswered::Closed); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // Certify view 1 so it remains a usable fallback after view 2 is // abandoned, then let the outgoing leader issue view 2. let mut fixture = HandoffFixture::start( &mut context, VoterOptions { handoff: Handoff::Vote(()), stall_proposals: !closed, ..Default::default() }, ) .await; wait_for_request( &context, &fixture.propose_requests, View::new(3), Viewable::view, ) .await; assert_eq!( request_parents(&fixture.propose_requests), [(View::new(3), View::new(2))] ); // Nothing observable marks the voter consuming a closed response, so give // it time to do so before the parent resolves. if closed { let (_, response) = take_proposal_response(&context, &fixture.handoff_responses).await; drop(response); context.sleep(SETTLE).await; } // Nullifying the captured parent enters view 3. The abandoned // request must be replaced immediately with one on certified view 1. let (_, nullification_2) = build_nullification( &peers(&fixture.schemes, fixture.local_index), Round::new(Epoch::new(333), View::new(2)), fixture.quorum(), ); fixture .mailbox .recovered(Certificate::Nullification(nullification_2)); let deadline = context.current() + Duration::from_secs(2); while fixture.propose_requests.lock().len() < 2 { assert!( context.current() < deadline, "proposal was not reissued on the certified fallback" ); context.sleep(Duration::from_millis(10)).await; } let expected = [(View::new(3), View::new(2)), (View::new(3), View::new(1))]; assert_eq!(request_parents(&fixture.propose_requests), expected); // After a closed response, view 3 votes for the replacement proposal on view 1 // before its leader timeout. if closed { loop { match fixture .batcher .recv() .await .expect("batcher must remain open") { batcher::Message::Constructed(Vote::Notarize(vote)) if vote.view() == View::new(3) => { assert_eq!(vote.proposal.parent, View::new(1)); break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(3) => { panic!("view 3 timed out instead of proposing on view 1"); } _ => {} } } } assert_handoff_metrics( &context.encode(), HANDOFF_ACTOR_METRICS, &[("Requested", 1)], &[("AncestrySuperseded", 1)], ); }); } /// A stalled proposal build on an abandoned outgoing tip must be /// superseded while the incoming term-start view is still live. #[test_traced] fn test_pipelined_handoff_reissues_stalled_proposal_after_parent_nullification() { reissue_handoff_after_parent_nullification(Unanswered::Stalled); } /// A handoff response that closes before its parent resolves forfeits the /// view only if that parent certifies. Nullifying the parent must still /// offer the proposal opportunity on the certified fallback. #[test_traced] fn test_pipelined_handoff_reissues_closed_proposal_after_parent_nullification() { reissue_handoff_after_parent_nullification(Unanswered::Closed); } /// The incoming leader's handoff build rests on an outgoing chain A1 to A3 that it verified /// and voted for. A conflicting notarization for B1 at the chain's first view invalidates that /// ancestry, so the voter cancels the build before any timeout, live or after a restart /// replays its votes. fn displaced_ancestor_cancels_before_timeout(restart: bool) { deterministic::Runner::timed(Duration::from_secs(5)).start(|mut context| async move { let n = 4; let epoch = Epoch::new(333); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"pipelined_handoff_displaced_ancestor", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(3)), Duration::from_secs(300), ViewDelta::new(2), ); let schedule: RoundRobinElector = elector.clone().build(schemes[0].participants()); let outgoing_index = usize::from(schedule.elect(Round::new(epoch, View::new(1)), ())); let local_index = usize::from(schedule.elect(Round::new(epoch, View::new(4)), ())); assert_eq!((outgoing_index, local_index), (2, 3)); let requests: ProposeRequests = Arc::new(Mutex::new(Vec::new())); let responses: HandoffResponses = Arc::new(Mutex::new(Vec::new())); let certifications: CertificationRequests = Arc::new(Mutex::new(Vec::new())); let actor_handle = Arc::new(Mutex::new(None)); let options = || VoterOptions { local_index, actor_handle: Some(actor_handle.clone()), leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(20), timeout_retry: Duration::from_secs(30), propose_requests: Some(requests.clone()), handoff_responses: Some(responses.clone()), handoff: Handoff::Stage(()), certifier: hold(View::new(1), certifications.clone()), ..Default::default() }; let (mut mailbox, mut batcher, mut _resolver, mut relay, mut reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector.clone(), options(), ) .await; // The local voter verifies the exact A-chain parents and votes without a certificate. let genesis = mocks::application::genesis::(epoch); let mut parent_payload = genesis; let mut proposals = Vec::new(); for view in 1..=3u64 { let round = Round::new(epoch, View::new(view)); let contents = (round, parent_payload, view).encode(); let proposal = Proposal::new( round, View::new(view - 1), Sha256::hash(&[contents.as_ref()]), ); relay.broadcast( &participants[outgoing_index], Recipients::All, (proposal.payload, contents), ); mailbox.proposal(proposal.clone()); loop { match batcher.recv().await.expect("batcher must remain open") { batcher::Message::Constructed(Vote::Notarize(vote)) => { assert_eq!(vote.proposal, proposal); assert_eq!( vote.attestation.signer, Participant::from_usize(local_index) ); assert!(vote.verify(&mut context, &schemes[local_index], &Sequential)); break; } batcher::Message::Constructed(Vote::Nullify(_)) => { panic!("local voter timed out before preparing the handoff"); } batcher::Message::Update { current, .. } => { assert_eq!(current, View::new(1)); } _ => {} } } parent_payload = proposal.payload; proposals.push(proposal); } let (candidate, mut response) = take_proposal_response(&context, &responses).await; assert!(matches!(candidate, Handoff::Stage(_))); assert!(!response.is_closed()); { let requests = requests.lock(); assert_eq!(requests.len(), 1); assert_eq!(requests[0].round, Round::new(epoch, View::new(4))); assert_eq!(requests[0].parent, (View::new(3), parent_payload)); } assert!(reporter.notarizations.lock().is_empty()); assert!(reporter.certifications.lock().is_empty()); if restart { let handle = actor_handle.lock().take().unwrap(); handle.abort(); assert!(handle.await.is_err()); assert!(response.is_closed()); requests.lock().clear(); let restarted_context = context.child("restarted"); (mailbox, batcher, _resolver, relay, reporter) = setup_voter( &restarted_context, &oracle, &participants, &schemes, elector, options(), ) .await; let (candidate, fresh_response) = take_proposal_response(&context, &responses).await; assert!(matches!(candidate, Handoff::Stage(_))); response = fresh_response; let requests = requests.lock(); assert_eq!(requests.len(), 1); assert_eq!(requests[0].round, Round::new(epoch, View::new(4))); assert_eq!(requests[0].parent, (View::new(3), parent_payload)); } // Only the outgoing leader equivocates: validators 0, 1, and 2 notarize B1, while // local 3 voted A1. let round = Round::new(epoch, View::new(1)); let contents = (round, genesis, 4u64).encode(); let conflicting = Proposal::new(round, View::zero(), Sha256::hash(&[contents.as_ref()])); assert_ne!(conflicting.payload, proposals[0].payload); let (votes, notarization) = build_notarization(&peers(&schemes, local_index), &conflicting, quorum(n)); assert_eq!(votes.len(), 3); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(local_index)) ); assert!(notarization.verify(&mut context, &schemes[local_index], &Sequential)); relay.broadcast( &participants[outgoing_index], Recipients::All, (conflicting.payload, contents), ); mailbox.recovered(Certificate::Notarization(notarization)); let certified = take_certification_request(&context, &certifications, View::new(1)).await; // Hold B1's certification: the build must be cancelled while it is still pending. response.closed().await; assert!(reporter.certifications.lock().is_empty()); assert_eq!( labeled_metric_snapshot( &context.encode(), if restart { "restarted_actor" } else { "actor" }, "handoff_abandoned", "reason", ) .get("AncestryInvalidated"), Some(&1), ); certified .send(true) .expect("B1 certification must remain pending"); loop { match batcher.recv().await.expect("batcher must remain open") { batcher::Message::Update { current, .. } if current == View::new(2) => break, batcher::Message::Constructed(Vote::Nullify(_)) => { panic!("B1 certification required a timeout"); } _ => {} } } assert_eq!( reporter .certifications .lock() .get(&View::new(1)) .map(|notarization| notarization.proposal.payload), Some(conflicting.payload), ); }); } #[test_traced] fn test_pipelined_handoff_displaced_ancestor_cancels_before_timeout() { displaced_ancestor_cancels_before_timeout(false); } #[test_traced] fn test_pipelined_handoff_displaced_ancestor_cancels_after_restart() { displaced_ancestor_cancels_before_timeout(true); } /// A view 4 handoff build pending on the uncertified final view of a three-view outgoing /// term. View 2 is notarized while the application holds its certification, so we wait /// in view 2, below the handoff parent. struct OutgoingTermHandoff { epoch: Epoch, quorum: u32, schemes: Vec, local_index: usize, mailbox: Mailbox, batcher: mailbox::Receiver>, proposals: Vec>, propose_requests: ProposeRequests, /// The application's held response to the view 4 handoff request. response: oneshot::Sender>, /// The application's held certification of view 2. certified: Option>, } impl OutgoingTermHandoff { async fn new(context: &mut deterministic::Context, namespace: &[u8]) -> Self { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let Fixture { participants, schemes, .. } = ed25519::fixture(context, namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Views 1 to 3 form the outgoing term, and our term starts at view 4. let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(3)), Duration::from_secs(300), ViewDelta::new(1), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(4)), ())); let outgoing_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let outgoing = participants[outgoing_index].clone(); let propose_requests = Arc::new(Mutex::new(Vec::new())); let handoff_responses = Arc::new(Mutex::new(Vec::new())); let certification_requests: CertificationRequests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher, _, relay, _) = setup_voter( &*context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(2), timeout_retry: Duration::from_secs(300), local_index, propose_requests: Some(propose_requests.clone()), handoff_responses: Some(handoff_responses.clone()), handoff: Handoff::Vote(()), certifier: hold(View::new(2), certification_requests.clone()), ..Default::default() }, ) .await; // The outgoing leader relays views 1 to 3, each built on the previous one. let mut parent_payload = mocks::application::genesis::(epoch); let mut proposals = Vec::new(); for view in 1..=3u64 { let proposal = Proposal::new( Round::new(epoch, View::new(view)), View::new(view - 1), Sha256::hash(&[format!("pipelined_handoff_parent_term_{view}").as_bytes()]), ); relay.broadcast( &outgoing, Recipients::All, ( proposal.payload, (proposal.round, parent_payload, view - 1).encode(), ), ); parent_payload = proposal.payload; proposals.push(proposal); } // View 1 certifies, and we vote for views 2 and 3. Our vote for the term's // final view starts a handoff build for view 4, which the application holds. let (_, notarization_1) = build_notarization(&peers(&schemes, local_index), &proposals[0], quorum); mailbox.recovered(Certificate::Notarization(notarization_1)); loop { if matches!( batcher.recv().await.expect("batcher must remain open"), batcher::Message::Update { current, .. } if current == View::new(2) ) { break; } } for proposal in &proposals[1..] { mailbox.proposal(proposal.clone()); loop { if let batcher::Message::Constructed(Vote::Notarize(vote)) = batcher.recv().await.expect("batcher must remain open") && vote.view() == proposal.view() { break; } } } let (_, response) = take_proposal_response(context, &handoff_responses).await; // View 2 notarizes, and the application holds its certification. let (_, notarization_2) = build_notarization(&schemes, &proposals[1], quorum); mailbox.recovered(Certificate::Notarization(notarization_2)); let certified = take_certification_request(context, &certification_requests, View::new(2)).await; Self { epoch, quorum, schemes, local_index, mailbox, batcher, proposals, propose_requests, response, certified: Some(certified), } } /// Lets the application certify view 2. fn certify_view_2(&mut self) { self.certified .take() .expect("view 2 certification must be held") .send(true) .unwrap(); } /// Waits until consensus cancels the pending handoff build. async fn wait_for_cancellation(&mut self, context: &deterministic::Context) { while !self.response.is_closed() { select! { _ = self.batcher.recv() => {}, _ = context.sleep(Duration::from_millis(1)) => {}, } } } /// Waits for our view 4 notarize vote and checks the parent it names. async fn wait_for_view_4_proposal(&mut self, parent: View) { loop { if let batcher::Message::Constructed(Vote::Notarize(vote)) = self.batcher.recv().await.expect("batcher must remain open") && vote.view() == View::new(4) { assert_eq!(vote.proposal.parent, parent); return; } } } } /// A nullification in the parent's term that arrives before our own nullify vote /// also cancels a pending handoff build. The application certifies the fallback /// parent only after that build stops, as when it verifies blocks behind an active /// build. Once the fallback certifies, the request is replaced with an ordinary /// request on it, and the term-start view proposes. #[test_traced] fn test_pipelined_handoff_parent_term_nullification_cancels_pending_build() { let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { let mut handoff = OutgoingTermHandoff::new( &mut context, b"pipelined_handoff_parent_term_nullification", ) .await; // The other validators nullify view 3 before our certification timeout, so we // enter view 4 without voting to nullify. let (votes, nullification_3) = build_nullification( &peers(&handoff.schemes, handoff.local_index), Round::new(handoff.epoch, View::new(3)), handoff.quorum, ); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(handoff.local_index)), "nullification quorum includes the local voter" ); handoff .mailbox .recovered(Certificate::Nullification(nullification_3)); handoff.wait_for_cancellation(&context).await; assert_eq!( request_parents(&handoff.propose_requests), [(View::new(4), View::new(3))] ); // The application now certifies view 2, so the request is replaced with an // ordinary request on view 2 and view 4 proposes before its leader timeout. handoff.certify_view_2(); handoff.wait_for_view_4_proposal(View::new(2)).await; assert_eq!( request_parents(&handoff.propose_requests), [(View::new(4), View::new(3)), (View::new(4), View::new(2))] ); assert_handoff_metrics( &context.encode(), "actor", &[("Requested", 1)], &[("AncestryInvalidated", 1), ("AncestrySuperseded", 1)], ); }); } /// A local nullify vote for the view we wait in also cancels a pending handoff build /// when that view is below the build's uncertified parent, since the build could keep /// that view from certifying. Once the view and the parent certify, the waiting request /// becomes an ordinary request for the same context. #[test_traced] fn test_pipelined_handoff_nullify_below_parent_cancels_pending_build() { let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { let mut handoff = OutgoingTermHandoff::new(&mut context, b"pipelined_handoff_nullify_below_parent") .await; // Our certification of view 2 times out, so we vote to nullify view 2, below the // handoff parent. wait_for_local_nullify(&mut handoff.batcher, View::new(2)).await; handoff.wait_for_cancellation(&context).await; assert_eq!( request_parents(&handoff.propose_requests), [(View::new(4), View::new(3))] ); // Views 2 and 3 then certify, so the waiting request becomes an ordinary request // for the same context, and view 4 proposes on view 3. handoff.certify_view_2(); let (_, notarization_3) = build_notarization(&handoff.schemes, &handoff.proposals[2], handoff.quorum); handoff .mailbox .recovered(Certificate::Notarization(notarization_3)); handoff.wait_for_view_4_proposal(View::new(3)).await; assert_eq!( request_parents(&handoff.propose_requests), [(View::new(4), View::new(3)), (View::new(4), View::new(3))] ); assert_handoff_metrics( &context.encode(), "actor", &[("Requested", 1)], &[("ParentNullify", 1)], ); }); } /// A waiting handoff is dropped when its parent certifies after we voted to /// nullify the handoff's view, because no proposal can be recorded there. /// /// With rotating terms and no certificate for view 1, nullifying the parent leaves /// no fallback parent, so the waiting request survives until the parent certifies. #[test_traced] fn test_pipelined_handoff_waiting_request_dropped_after_view_nullify() { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"pipelined_handoff_view_nullify", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default(); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let local_index = usize::from(built_elector.elect(Round::new(epoch, View::new(4)), ())); let propose_requests = Arc::new(Mutex::new(Vec::new())); let certification_requests: CertificationRequests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), local_index, propose_requests: Some(propose_requests.clone()), certifier: hold(View::new(3), certification_requests.clone()), ..Default::default() }, ) .await; let requests_for_4 = || { propose_requests .lock() .iter() .filter(|request| request.view() == View::new(4)) .count() }; // Enter view 3 without a certificate for view 1. let peers = peers(&schemes, local_index); let (votes, nullification_2) = build_nullification(&peers, Round::new(epoch, View::new(2)), quorum); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(local_index)), "nullification quorum includes the local voter" ); mailbox.recovered(Certificate::Nullification(nullification_2)); // Notarize view 3 and hold its certification. The application answers the view 4 // handoff with `Wait`, so its proposal waits until view 3 certifies. let proposal_3 = Proposal::new( Round::new(epoch, View::new(3)), View::zero(), Sha256::hash(&[b"pipelined_handoff_view_nullify_3"]), ); let (votes, notarization_3) = build_notarization(&peers, &proposal_3, quorum); assert!( votes .iter() .all(|vote| vote.attestation.signer != Participant::from_usize(local_index)), "unavailable local view-3 notarize share" ); mailbox.recovered(Certificate::Notarization(notarization_3)); let certified = take_certification_request(&context, &certification_requests, View::new(3)).await; wait_for_request(&context, &propose_requests, View::new(4), Viewable::view).await; // Nullifying view 3 enters view 4 with the request retained, and our leader // timeout then nullifies view 4. let (_, nullification_3) = build_nullification(&peers, Round::new(epoch, View::new(3)), quorum); mailbox.recovered(Certificate::Nullification(nullification_3)); wait_for_local_nullify(&mut batcher, View::new(4)).await; // View 3 certifies while view 4 is still current. certified.send(true).unwrap(); let deadline = context.current() + Duration::from_secs(1); loop { select! { _ = batcher.recv() => {}, _ = context.sleep_until(deadline) => break, } } assert_eq!(requests_for_4(), 1, "no build follows our nullify vote"); assert_handoff_metrics( &context.encode(), "actor", &[("WaitReturned", 1), ("Requested", 1)], &[("ViewNullify", 1)], ); }); } /// A follower that entered a term start through a nullification of the outgoing /// tip requests the tip's certificate from the incoming leader, which built on it. #[test_traced] fn test_pipelined_handoff_parent_repair_targets_leader() { let n = 5; let quorum = quorum(n); let namespace = b"pipelined_handoff_parent_repair_targets_leader".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(2)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(1), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let outgoing_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let incoming_index = usize::from(built_elector.elect(Round::new(epoch, View::new(3)), ())); let local_index = (0..participants.len()) .find(|index| *index != outgoing_index && *index != incoming_index) .expect("a follower must exist"); let outgoing = participants[outgoing_index].clone(); let incoming = participants[incoming_index].clone(); let (mut mailbox, mut batcher_receiver, mut resolver_receiver, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), local_index, ..Default::default() }, ) .await; let proposal_2 = prepare_pipelined_handoff_tip( &schemes, local_index, &outgoing, &mut mailbox, &mut batcher_receiver, &mut resolver_receiver, &relay, ) .await; // The follower holds a nullification of the tip that the incoming // leader built on, so it enters view 3 without the tip's certificate. let (_, nullification_2) = build_nullification( &peers(&schemes, local_index), Round::new(epoch, View::new(2)), quorum, ); mailbox.recovered(Certificate::Nullification(nullification_2)); let proposal_3 = Proposal::new( Round::new(epoch, View::new(3)), View::new(2), Sha256::hash(&[b"parent_repair_view_3"]), ); relay.broadcast( &incoming, Recipients::All, ( proposal_3.payload, (proposal_3.round, proposal_2.payload, 2u64).encode(), ), ); mailbox.proposal(proposal_3); loop { let MailboxMessage::Resolve { proposal, view, kind, target, .. } = resolver_receiver.recv().await.unwrap() else { continue; }; assert_eq!(proposal, View::new(3)); assert_eq!(view, View::new(2)); assert!(matches!(kind, Kind::Notarization)); assert_eq!(target, Some(incoming.clone())); break; } // Deliver the parent as an unrestricted resolver response. The // follower must certify it, verify the buffered child, and vote // before the term-start view times out. let (_, notarization_2) = build_notarization(&schemes, &proposal_2, quorum); mailbox.recovered(Certificate::Notarization(notarization_2)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == View::new(3) => { break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(3) => { panic!("term-start view timed out before parent repair completed"); } batcher::Message::Update { .. } | batcher::Message::Constructed(_) => {} } } }); } /// A directly notarized descendant must not hide a local certification /// failure in its same-term ancestry. #[test_traced] fn test_failed_certification_blocks_direct_descendant_verify() { let n = 5; let quorum = quorum(n); let namespace = b"failed_certification_blocks_direct_descendant_verify".to_vec(); let epoch = Epoch::new(333); let term_length = TermLength::new(NZU32!(5)); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // Build a stable-leader network with a follower whose application // rejects certification for view 1. let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default().with_term( term_length, Duration::from_secs(30), ViewDelta::new(2), ); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let leader_idx = built_elector.elect(Round::new(epoch, View::new(1)), ()); let local_index = (usize::from(leader_idx) + 1) % participants.len(); let leader = participants[usize::from(leader_idx)].clone(); let verify_requests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(30), local_index, verify_requests: Some(verify_requests.clone()), certifier: mocks::application::Certifier::Custom(Box::new(|round, _| { round.view() != View::new(1) })), ..Default::default() }, ) .await; assert!(matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Update { .. } )); // Locally notarize view 1 so its child enters the optimistic // issuance window. let genesis = mocks::application::genesis::(epoch); let proposal_1 = Proposal::new( Round::new(epoch, View::new(1)), View::zero(), Sha256::hash(&[b"failed_ancestor_view_1"]), ); let contents_1 = (proposal_1.round, genesis, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_1.payload, contents_1)); mailbox.proposal(proposal_1.clone()); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = msg.unwrap() && notarize.view() == View::new(1) { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected notarize for view 1"); } } } // Extend the chain optimistically before view 1's certificate // arrives. let proposal_2 = Proposal::new( Round::new(epoch, View::new(2)), View::new(1), Sha256::hash(&[b"failed_ancestor_view_2"]), ); let contents_2 = (proposal_2.round, proposal_1.payload, 1u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_2.payload, contents_2)); mailbox.proposal(proposal_2.clone()); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = msg.unwrap() && notarize.view() == View::new(2) { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected optimistic notarize for view 2"); } } } // Prepare both certificates, but deliver only view 1. Its // certification rejection must emit a nullify before view 2 // becomes directly notarized. let (_, notarization_1) = build_notarization(&schemes, &proposal_1, quorum); let (_, notarization_2) = build_notarization(&schemes, &proposal_2, quorum); mailbox.recovered(Certificate::Notarization(notarization_1)); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Nullify(nullify)) = msg.unwrap() && nullify.view() == View::new(1) { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected failed certification to nullify view 1"); } } } // Deliver view 2's certificate and extend the direct chain. The // unresolved view-1 rejection must block view 3 before application // verification. mailbox.recovered(Certificate::Notarization(notarization_2)); let proposal_3 = Proposal::new( Round::new(epoch, View::new(3)), View::new(2), Sha256::hash(&[b"failed_ancestor_view_3"]), ); let contents_3 = (proposal_3.round, proposal_2.payload, 2u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_3.payload, contents_3)); mailbox.proposal(proposal_3); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Notarize(notarize)) = msg.unwrap() && notarize.view() == View::new(3) { panic!("rejected ancestry must not produce a view 3 notarize"); } }, _ = context.sleep(Duration::from_secs(1)) => break, } } assert!( !verify_requests.lock().contains(&View::new(3)), "rejected ancestry must not reach application verification" ); // Nullify the failed ancestor to abandon the term and confirm the // voter can advance. let (_, nullification) = build_nullification(&schemes, Round::new(epoch, View::new(1)), quorum); mailbox.recovered(Certificate::Nullification(nullification)); loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current == View::new(6) { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected nullification to advance to the next term"); } } } }); } /// A recovered finalization above the current view must be applied and /// reported. fn recovered_finalization_is_reported(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"recovered_finalization_is_reported".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let (mut mailbox, mut batcher_receiver, _, _, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions::default(), ) .await; // Wait for batcher to be notified let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, leader: _, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Recover a finalization for view 2 (one above the current view). let view = View::new(2); let proposal = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"recovered_finalization_is_reported"]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization.clone())); // Wait for batcher to be notified of finalization loop { let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { finalized, .. } if finalized == view => break, _ => continue, } } // Verify finalization was recorded by checking reporter let finalizations = reporter.finalizations.lock(); let (recorded, _) = finalizations .get(&view) .expect("finalization should be recorded"); assert_eq!(recorded, &finalization); }); } #[test_traced] fn test_recovered_finalization_is_reported() { recovered_finalization_is_reported::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); recovered_finalization_is_reported::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); recovered_finalization_is_reported::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); recovered_finalization_is_reported::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); recovered_finalization_is_reported::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); recovered_finalization_is_reported::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Regression: a voter that misses one mid-term notarization must request /// it from the resolver. Certification of every later view in the term /// requires the exact-view certificate, so without a fetch the voter can /// never certify or advance again. A quorum-critical voter wedged this /// way halts the network. fn missed_notarization_is_fetched(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 4; let quorum = quorum(n); let namespace = b"missed_notarization_is_fetched".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Stable leader with optimistic validation enabled. Participant 0 // (the voter under test) is a follower for term 1. let elector = RoundRobin::::default().with_term( TermLength::new(NZU32!(5)), Duration::from_secs(30), ViewDelta::new(2), ); let (mut mailbox, mut batcher_receiver, mut resolver_receiver, _, _reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions::default(), ) .await; // Build the term's proposal chain. let epoch = Epoch::new(333); let proposal = |view: u64, parent: u64| { Proposal::new( Round::new(epoch, View::new(view)), View::new(parent), Sha256::hash(&[&view.to_be_bytes()]), ) }; let (_, notarization_1) = build_notarization(&schemes, &proposal(1, 0), quorum); let (_, notarization_2) = build_notarization(&schemes, &proposal(2, 1), quorum); let (_, notarization_3) = build_notarization(&schemes, &proposal(3, 2), quorum); // Deliver notarization(1) and wait for the voter to certify it and // enter view 2. mailbox.recovered(Certificate::Notarization(notarization_1)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, .. } if current == View::new(2) => break, _ => continue, } } while resolver_receiver.recv().now_or_never().flatten().is_some() {} // Deliver notarization(3) without notarization(2). The voter saw // the certificate for view 3 but missed the one for view 2. // Certifying view 3 requires the exact certificate for view 2, so // the voter asks any validator for it. mailbox.recovered(Certificate::Notarization(notarization_3)); assert!(matches!( resolver_receiver.recv().await.unwrap(), MailboxMessage::Updated { certificate: Certificate::Notarization(notarization), .. } if notarization.view() == View::new(3) )); loop { select! { message = resolver_receiver.recv() => { let MailboxMessage::Resolve { proposal, view, kind, target, .. } = message.unwrap() else { continue; }; assert_eq!(proposal, View::new(3)); assert_eq!(view, View::new(2)); assert!(matches!(kind, Kind::Notarization)); assert!(target.is_none()); break; }, _ = context.sleep(Duration::from_secs(20)) => { panic!("voter never requested the missed notarization"); }, } } // Deliver the fetched parent. The voter must certify the parent // before the child and then resume finalize voting. while batcher_receiver.recv().now_or_never().flatten().is_some() {} mailbox.recovered(Certificate::Notarization(notarization_2)); let mut certified = Vec::new(); let mut finalized = Vec::new(); loop { select! { message = resolver_receiver.recv() => match message.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == View::new(2) || notarization.view() == View::new(3) => { assert!(success); certified.push(notarization.view()); } MailboxMessage::Updated { .. } | MailboxMessage::Certified { .. } | MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => {} }, message = batcher_receiver.recv() => { if let batcher::Message::Constructed(Vote::Finalize(vote)) = message.unwrap() && (vote.view() == View::new(2) || vote.view() == View::new(3)) { finalized.push(vote.view()); } }, _ = context.sleep(Duration::from_secs(20)) => { panic!( "recovery stalled: certified={certified:?}, finalized={finalized:?}" ); }, } if certified == [View::new(2), View::new(3)] && finalized.contains(&View::new(3)) { break; } } }); } #[test_traced] fn test_missed_notarization_is_fetched() { // Request routing is scheme-independent; one scheme is enough. missed_notarization_is_fetched(ed25519::fixture); } /// Tests that when proposal verification fails, the voter emits a nullify vote /// immediately rather than waiting for the timeout. fn verification_failure_emits_nullify_immediately(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let view_retention = ViewDelta::new(10); let executor = deterministic::Runner::timed(Duration::from_secs(5)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Use participant[0] as the voter let signing = schemes[0].clone(); let me = participants[0].clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (10.0, 0.0), // 10ms verification latency certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); // Configure application to always fail verification actor.set_fail_verification(true); actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector: elector.clone().build(signing.participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_verify_fail_test_{me}"), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), // Use long timeouts to prove nullify comes immediately, not from timeout leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention, replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); // Resolver and batcher mailboxes let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(2)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(16)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); // Register network channels for the validator let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); // Start the actor voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Wait for initial batcher update let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { .. } => {} _ => panic!("expected Update message"), } // Advance views until we find one where we're NOT the leader (so we verify // rather than propose). Keep track of the previous view's proposal for parent. let mut current_view = View::new(1); let mut prev_proposal = Proposal::new( Round::new(Epoch::new(333), current_view), View::zero(), Sha256::hash(&[b"v0"]), ); let (target_view, leader) = loop { // Send finalization to advance to next view let (_, finalization) = build_finalization(&schemes, &prev_proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for the view update let (new_view, leader) = loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { if current > current_view { break (current, leader); } } batcher::Message::Constructed(_) => {} } }; current_view = new_view; // Check if we're NOT the leader for this view if leader != Participant::new(0) { break (current_view, participants[usize::from(leader)].clone()); } // We're the leader, advance to next view prev_proposal = Proposal::new( Round::new(Epoch::new(333), current_view), current_view.previous().unwrap(), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); }; // Create proposal for the target view (where we are a verifier) let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"test_proposal"]), ); // Broadcast the payload contents so verification can complete (the automaton waits // for the contents via the relay). let parent_payload = Sha256::hash(&[target_view .previous() .unwrap() .get() .to_be_bytes() .as_slice()]); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal); // Wait for nullify vote for target_view. Since timeouts are 10s, receiving it // within 1s proves it came from verification failure, not timeout. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected nullify for view {} within 1s (timeouts are 10s)", target_view ); }, } } }); } #[test_traced] fn test_verification_failure_emits_nullify_immediately() { verification_failure_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); verification_failure_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); verification_failure_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); verification_failure_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); verification_failure_emits_nullify_immediately::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); verification_failure_emits_nullify_immediately::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Tests that a leader-nullify timeout hint fast-paths local nullify construction. fn leader_nullify_timeout_hint_fast_paths_nullify(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"leader_nullify_timeout_hint_fast_paths_nullify".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(5)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let me_idx = Participant::new(0); let signing = schemes[0].clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector: elector.clone().build(signing.participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_leader_nullify_fast_path_{me}"), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), // Long timeouts prove nullify came from fast-path, not timer expiry. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(32)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); let (mut current_view, mut current_leader) = match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => (current, leader), _ => panic!("expected initial update"), }; // Move to a non-leader view so we act as a verifier. while current_leader == me_idx { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } if current > current_view => { current_view = current; current_leader = leader; break; } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } } let target_view = current_view; mailbox.timeout( Round::new(Epoch::new(333), target_view), TimeoutReason::LeaderNullify, ); // Expect local nullify quickly despite 10s timeouts. loop { select! { message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Constructed(_) => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected nullify for view {} within 1s (timeouts are 10s)", target_view ); }, } } // Send the same expire signal again. Duplicates should not retrigger the fast-path. mailbox.timeout( Round::new(Epoch::new(333), target_view), TimeoutReason::LeaderNullify, ); let duplicate_window = context.current() + Duration::from_millis(300); loop { select! { _ = context.sleep_until(duplicate_window) => break, message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!("duplicate leader nullify should not retrigger fast-path"); } batcher::Message::Constructed(_) => {} }, } } }); } #[test_traced] fn test_leader_nullify_timeout_hint_fast_paths_nullify() { leader_nullify_timeout_hint_fast_paths_nullify::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); leader_nullify_timeout_hint_fast_paths_nullify::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); leader_nullify_timeout_hint_fast_paths_nullify::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); leader_nullify_timeout_hint_fast_paths_nullify::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); leader_nullify_timeout_hint_fast_paths_nullify::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); leader_nullify_timeout_hint_fast_paths_nullify::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Tests that if the application drops proposal requests, the leader emits `nullify` /// immediately instead of waiting for timeout. fn dropped_propose_emits_nullify_immediately(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"dropped_propose_emits_nullify_immediately".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let me_idx = Participant::new(0); let signing = schemes[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.set_drop_proposals(true); app_actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector: elector.clone().build(signing.participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_drop_propose_test_{me}"), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), // Long timeouts prove nullify came from fast-path, not timer expiry. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(32)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); let (mut current_view, mut current_leader) = match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => (current, leader), _ => panic!("expected initial update"), }; // Move to a leader view. while current_leader != me_idx { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } if current > current_view => { current_view = current; current_leader = leader; break; } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } } let target_view = current_view; // With 10s timeouts, seeing nullify within 1s proves we fast-pathed on dropped propose. loop { select! { message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Constructed(_) => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected nullify for view {} within 1s (timeouts are 10s)", target_view ); }, } } }); } #[test_traced] fn test_dropped_propose_emits_nullify_immediately() { dropped_propose_emits_nullify_immediately(bls12381_threshold_vrf::fixture::); dropped_propose_emits_nullify_immediately(bls12381_threshold_vrf::fixture::); dropped_propose_emits_nullify_immediately(bls12381_multisig::fixture::); dropped_propose_emits_nullify_immediately(bls12381_multisig::fixture::); dropped_propose_emits_nullify_immediately(ed25519::fixture); dropped_propose_emits_nullify_immediately(secp256r1::fixture); } /// Tests that if the application drops verification requests, the voter emits `nullify` /// immediately instead of waiting for timeout. fn dropped_verify_emits_nullify_immediately(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"dropped_verify_emits_nullify_immediately".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let signing = schemes[0].clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); app_actor.set_drop_verifications(true); app_actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector: elector.clone().build(signing.participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_drop_verify_test_{me}"), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), // Use long timeouts so a fast nullify proves we did not wait for timeout. leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(32)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); // Initial batcher update. match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => {} _ => panic!("expected initial update"), } // Find a view where we are a verifier (not leader). let mut current_view = View::new(1); let (target_view, leader) = loop { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); let (new_view, leader) = loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { if current > current_view { break (current, leader); } } batcher::Message::Constructed(_) => {} } }; current_view = new_view; if leader != Participant::new(0) { break (current_view, participants[usize::from(leader)].clone()); } }; // Trigger verification in target view. The application drops the verify response, // which should immediately trigger nullify. let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap(), Sha256::hash(&[b"drop_verify"]), ); let contents = ( proposal.round, Sha256::hash(&[target_view .previous() .unwrap() .get() .to_be_bytes() .as_slice()]), 7u64, ) .encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal); // With 10s timeouts, seeing nullify within 1s proves we fast-pathed on dropped verify. loop { select! { message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Constructed(_) => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected nullify for view {} within 1s (timeouts are 10s)", target_view ); }, } } // Ensure dropped verify maps to the expected timeout reason metric. let encoded = context.encode(); assert!( encoded.lines().any(|line| { line.contains("_timeouts") && line.contains("reason=\"IgnoredProposal\"") && !line.ends_with(" 0") }), "expected non-zero timeout metric with reason=IgnoredProposal" ); }); } #[test_traced] fn test_dropped_verify_emits_nullify_immediately() { dropped_verify_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); dropped_verify_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); dropped_verify_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); dropped_verify_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); dropped_verify_emits_nullify_immediately::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); dropped_verify_emits_nullify_immediately::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// A view-exiting certificate cancels an in-flight verification before /// its completion can be processed into a vote for the exited view. #[test_traced] fn test_finalization_cancels_inflight_verification() { let n = 4; let quorum = quorum(n); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"finalization_cancels_verify", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default(); let built_elector: RoundRobinElector = elector.clone().build(schemes[0].participants()); let leader_index = usize::from(built_elector.elect(Round::new(epoch, View::new(1)), ())); let local_index = (leader_index + 1) % schemes.len(); let verify_requests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher_receiver, _, relay, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), local_index, verify_latency_ms: 1_000.0, verify_requests: Some(verify_requests.clone()), ..Default::default() }, ) .await; let (target_view, leader) = match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => (current, leader), _ => panic!("expected initial update"), }; assert_eq!(target_view, View::new(1)); assert_eq!(usize::from(leader), leader_index); assert_ne!(usize::from(leader), local_index); let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap_or(View::zero()), Sha256::hash(&[b"verification_overtaken_by_finalization"]), ); let contents = ( proposal.round, mocks::application::genesis::(epoch), 7u64, ) .encode(); relay.broadcast( &participants[leader_index], Recipients::All, (proposal.payload, contents), ); mailbox.proposal(proposal.clone()); context.sleep(Duration::from_millis(10)).await; assert!( verify_requests.lock().contains(&target_view), "verification must be in flight before finalization arrives" ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, .. } if current > target_view => break, _ => {} } } // Let the application finish the now-canceled verification. The // old view must not regain a path into vote construction. context.sleep(Duration::from_secs(2)).await; assert!( !reporter.notarizes.lock().contains_key(&target_view), "verification completed after finalization must not emit a notarize vote" ); }); } /// Tests that permanently invalid proposal ancestry fast-paths `nullify` /// instead of waiting for the local timeout. fn invalid_ancestry_emits_nullify_immediately(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"invalid_ancestry_emits_nullify_immediately".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_mins(60), ..Default::default() }, ) .await; // Advance until we are a verifier in a post-finalization view. let me = Participant::new(0); let (mut current_view, mut current_leader) = match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => (current, leader), _ => panic!("expected initial update"), }; while current_view == View::new(1) || current_leader == me { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } if current > current_view => { current_view = current; current_leader = leader; break; } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } } // Inject a proposal whose parent is below the finalized floor. let target_view = current_view; let invalid_parent = target_view .previous() .expect("target view must have a finalized predecessor") .previous() .unwrap_or(View::zero()); let proposal = Proposal::new( Round::new(epoch, target_view), invalid_parent, Sha256::hash(&[b"invalid_parent_before_finalized"]), ); mailbox.proposal(proposal); // With 10s timeouts, seeing nullify within 1s proves we fast-pathed on invalid ancestry. loop { select! { message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Constructed(_) => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!( "expected nullify for view {} within 1s (timeouts are 10s)", target_view ); }, } } // Ensure invalid ancestry maps to the expected timeout reason metric. let encoded = context.encode(); assert!( encoded.lines().any(|line| { line.contains("_timeouts") && line.contains("reason=\"InvalidProposal\"") && !line.ends_with(" 0") }), "expected non-zero timeout metric with reason=InvalidProposal" ); }); } #[test_traced] fn test_invalid_ancestry_emits_nullify_immediately() { invalid_ancestry_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); invalid_ancestry_emits_nullify_immediately::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); invalid_ancestry_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); invalid_ancestry_emits_nullify_immediately::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); invalid_ancestry_emits_nullify_immediately::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); invalid_ancestry_emits_nullify_immediately::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Tests that a later dropped verification still yields network voting after /// prior successful participation. fn dropped_verify_still_votes_after_prior_participation(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"dropped_verify_still_votes_after_prior_participation".to_vec(); let epoch = Epoch::new(333); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let me_idx = Participant::new(0); let signing = schemes[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: signing.clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), application_cfg); app_actor.set_drop_verifications(true); app_actor.start(); let voter_cfg = Config { scheme: signing.clone(), elector: elector.clone().build(signing.participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: format!("voter_dropped_verify_after_participation_{me}"), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(250), certification_timeout: Duration::from_millis(250), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(10240), write_buffer: NZUsize!(10240), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let resolver_mailbox = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(64)); let batcher_mailbox = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let observer = participants[1].clone(); let (_, mut observer_vote_receiver) = oracle .control(observer.clone()) .register(0, TEST_QUOTA) .await .unwrap(); oracle .add_link( me.clone(), observer, Link { latency: Duration::from_millis(0), jitter: Duration::from_millis(0), success_rate: probability!(1.0), }, ) .await .unwrap(); voter.start( batcher_mailbox, resolver_mailbox, vote_sender, certificate_sender, ); let (mut current_view, mut current_leader) = match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => (current, leader), _ => panic!("expected initial update"), }; // Move to a view where we are leader. while current_leader != me_idx { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } if current > current_view => { current_view = current; current_leader = leader; break; } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } } let leader_view = current_view; // Wait until we actually broadcast a vote in our leader view. let ready_deadline = context.current() + Duration::from_secs(1); let mut became_ready = false; loop { select! { _ = context.sleep_until(ready_deadline) => break, message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} }, message = commonware_p2p::Receiver::recv(&mut observer_vote_receiver) => { let (_, message) = message.unwrap(); let vote: Vote = Vote::decode(message).unwrap(); if vote.view() == leader_view { became_ready = true; break; } }, } } assert!( became_ready, "expected a network vote in leader view {leader_view}" ); // Move to a non-leader view to trigger dropped verification. let (target_view, target_leader) = loop { let proposal = Proposal::new( Round::new(epoch, current_view), current_view.previous().unwrap_or(View::zero()), Sha256::hash(&[¤t_view.get().to_be_bytes()]), ); let (_, finalization) = build_finalization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Finalization(finalization)); let mut found = None; loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } if current > current_view => { current_view = current; if leader != me_idx { found = Some((current, participants[usize::from(leader)].clone())); } break; } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } if let Some(target) = found { break target; } }; // This verify request will be dropped by the application. let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap(), Sha256::hash(&[b"drop_verify_after_ready"]), ); let contents = ( proposal.round, Sha256::hash(&[target_view .previous() .unwrap() .get() .to_be_bytes() .as_slice()]), 11u64, ) .encode(); relay.broadcast( &target_leader, Recipients::All, (proposal.payload, contents), ); mailbox.proposal(proposal); // We should still broadcast for target_view (typically a nullify after timeout). let target_deadline = context.current() + Duration::from_secs(1); let mut saw_target_network_vote = false; loop { select! { _ = context.sleep_until(target_deadline) => break, message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} }, message = commonware_p2p::Receiver::recv(&mut observer_vote_receiver) => { let (_, message) = message.unwrap(); let vote: Vote = Vote::decode(message).unwrap(); if vote.view() == target_view { saw_target_network_vote = true; break; } }, } } assert!( saw_target_network_vote, "expected a network vote for target view {target_view} after dropped verification" ); }); } #[test_traced] fn test_dropped_verify_still_votes_after_prior_participation() { dropped_verify_still_votes_after_prior_participation( bls12381_threshold_vrf::fixture::, ); dropped_verify_still_votes_after_prior_participation( bls12381_threshold_vrf::fixture::, ); dropped_verify_still_votes_after_prior_participation( bls12381_multisig::fixture::, ); dropped_verify_still_votes_after_prior_participation( bls12381_multisig::fixture::, ); dropped_verify_still_votes_after_prior_participation(ed25519::fixture); dropped_verify_still_votes_after_prior_participation(secp256r1::fixture); } /// Tests that after replay, we do not re-certify views that have already /// been certified or finalized. Tests both scenarios in the same journal: /// 1. Finalization at view 2 (certify never called) /// 2. Notarization at view 3 with certification (certify called once) /// /// After restart, certify should not be called for either view. fn no_recertification_after_replay(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"no_recertification_after_replay".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Track certify calls across restarts let certify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let tracker = certify_calls.clone(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let me = participants[0].clone(); // Create application with certify tracking let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new(move |_, d| { tracker.lock().push(d); true })), }; let (actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "no_recertification_after_replay".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for initial batcher notification if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Step 1: Send finalization for view 2 (certify should NOT be called) let view2 = View::new(2); let proposal2 = Proposal::new( Round::new(Epoch::new(333), view2), View::new(1), Sha256::hash(&[b"finalized_payload"]), ); let (_, finalization) = build_finalization(&schemes, &proposal2, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for finalization loop { if let batcher::Message::Update { finalized, .. } = batcher_receiver.recv().await.unwrap() && finalized >= view2 { break; } } assert_eq!( certify_calls.lock().len(), 0, "certify should not be called for finalization" ); // Step 2: Send notarization for view 3 (certify SHOULD be called) let view3 = View::new(3); let digest3 = Sha256::hash(&[b"payload_for_certification"]); let proposal3 = Proposal::new(Round::new(Epoch::new(333), view3), view2, digest3); // Broadcast payload and send proposal let contents = (proposal3.round, proposal2.payload, 0u64).encode(); relay.broadcast(&me, Recipients::All, (digest3, contents)); mailbox.proposal(proposal3.clone()); // Send notarization let (_, notarization) = build_notarization(&schemes, &proposal3, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait for view advancement (certification complete) loop { if let batcher::Message::Update { current, .. } = batcher_receiver.recv().await.unwrap() && current > view3 { break; } } assert_eq!( certify_calls.lock().len(), 1, "certify should be called once for notarization" ); // Restart voter handle.abort(); // Create new application with same tracker let tracker = certify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new(move |_, d| { tracker.lock().push(d); true })), }; let (actor, application) = mocks::application::Application::new( context.child("app").with_attribute("index", 2), app_cfg, ); actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "no_recertification_after_replay".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1000), timeout_retry: Duration::from_secs(1000), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for replay to complete if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Give time for any erroneous certification attempts context.sleep(Duration::from_millis(100)).await; // Verify no additional certify calls after replay assert_eq!( certify_calls.lock().len(), 1, "certify should not be called again after replay" ); }); } #[test_traced] fn test_no_recertification_after_replay() { no_recertification_after_replay::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); no_recertification_after_replay::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); no_recertification_after_replay::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); no_recertification_after_replay::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); no_recertification_after_replay::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); no_recertification_after_replay::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// When the voter is the leader of a view and builds its own proposal, it /// must not subsequently ask the automaton to verify that same proposal. /// /// This is guarded by `Slot::record_verified` (which sets `status = Verified` and /// `requested_verify = true`) and by `Round::verify_ready` short-circuiting /// for leader-owned views. This test asserts the end-to-end invariant on /// the live path (no restart): after calling `automaton.propose`, the voter /// must never call `automaton.verify` for the produced payload. fn no_self_verify_when_proposing(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"no_self_verify_when_proposing".to_vec(); let partition = "no_self_verify_when_proposing".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Install propose + verify observers from the start so we can assert the // leader's propose call fires but no verify call is issued for our proposal. let propose_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let verify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let propose_tracker = propose_calls.clone(); let verify_tracker = verify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor .set_propose_observer(Box::new(move |ctx| propose_tracker.lock().push(ctx.view()))); app_actor.set_verify_observer(Box::new(move |ctx, _| { verify_tracker.lock().push(ctx.view()) })); app_actor.start(); // Build and start the voter wired to the observing application. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_secs(1), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance to the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Wait for the voter to time out (leader_timeout) and construct a Nullify // for the view. By that point the voter has already run the full propose // flow (so propose_observer should have fired) and any spurious verify for // the same payload would have fired before the timeout. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Assert the live invariant: propose fired (precondition proving the leader // flow ran) but verify did not fire for the payload we just built. let proposed = propose_calls.lock(); let verified = verify_calls.lock(); assert!( proposed.contains(&target_view), "test precondition: voter must have called automaton.propose for the leader-owned view (observed: {proposed:?})" ); assert!( !verified.contains(&target_view), "voter must not verify its own leader-built proposal (observed: {verified:?})" ); }); } #[test_traced] fn test_no_self_verify_when_proposing() { no_self_verify_when_proposing(bls12381_threshold_vrf::fixture::); no_self_verify_when_proposing(bls12381_threshold_vrf::fixture::); no_self_verify_when_proposing(bls12381_multisig::fixture::); no_self_verify_when_proposing(bls12381_multisig::fixture::); no_self_verify_when_proposing(ed25519::fixture); no_self_verify_when_proposing(secp256r1::fixture); } /// Restart analogue of `no_self_verify_when_proposing`: after the voter has /// proposed and journaled a local notarize as leader, restarting must not /// cause the voter to re-propose or to verify its own proposal when it is /// re-delivered (e.g. via peer echo through the automaton). /// /// Replay of the journaled local notarize must restore the slot's proposal, /// `requested_build`, and `requested_verify` flags so that: /// - `should_build` returns false on the next run-loop iteration (no new /// `automaton.propose` call for a payload the voter already proposed /// and voted on pre-crash), and /// - subsequent delivery of the same proposal is a no-op (no /// `automaton.verify` call for a payload the voter built itself). fn no_self_propose_or_verify_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"no_self_propose_or_verify_after_restart".to_vec(); let partition = "no_self_propose_or_verify_after_restart".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Pre-restart: plain application (no observers) so the voter can // cleanly propose and journal its own notarize vote for view 2. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); // Build and start the pre-restart voter. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_secs(1), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance to the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Wait for the voter to emit its own notarize (journaled). The captured // proposal is reused post-restart to exercise the re-delivery path. let proposal = loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { break notarize.proposal; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } }; // Restart: abort the voter and construct a fresh application with // propose + verify observers to catch any spurious work for the // leader-owned view that has a journaled local notarize vote. handle.abort(); let propose_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let verify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let propose_tracker = propose_calls.clone(); let verify_tracker = verify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new( context.child("app_restarted"), app_cfg, ); app_actor .set_propose_observer(Box::new(move |ctx| propose_tracker.lock().push(ctx.view()))); app_actor.set_verify_observer(Box::new(move |ctx, _| { verify_tracker.lock().push(ctx.view()) })); app_actor.start(); // Build and start the post-restart voter against the same journal partition. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_secs(1), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for replay to complete; confirm we re-entered the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { assert_eq!(current, target_view); assert_eq!(leader, Participant::new(0)); break; } batcher::Message::Constructed(_) => {} } } // Re-deliver the proposal via the automaton, simulating peer echo after // restart. Any spurious propose/verify for this view would fire on the next // run-loop iteration. Then wait for leader_timeout to fire a Nullify, // proving the voter ran its full flow without ever advancing. mailbox.proposal(proposal.clone()); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Assert the restart invariant: neither propose nor verify fires post-restart // for a leader-owned view whose journaled notarize replay should have // restored the slot's proposal state. let proposed = propose_calls.lock(); let verified = verify_calls.lock(); assert!( !proposed.contains(&target_view), "voter must not propose for a leader-owned view after restart (observed: {proposed:?})" ); assert!( !verified.contains(&target_view), "voter must not verify its own leader-built proposal after restart (observed: {verified:?})" ); }); } #[test_traced] fn test_no_self_propose_or_verify_after_restart() { no_self_propose_or_verify_after_restart(bls12381_threshold_vrf::fixture::); no_self_propose_or_verify_after_restart(bls12381_threshold_vrf::fixture::); no_self_propose_or_verify_after_restart(bls12381_multisig::fixture::); no_self_propose_or_verify_after_restart(bls12381_multisig::fixture::); no_self_propose_or_verify_after_restart(ed25519::fixture); no_self_propose_or_verify_after_restart(secp256r1::fixture); } /// Regression: a leader that crashes after calling `automaton.propose` but /// before journaling its local `Notarize` must, on restart, issue at most a /// single `automaton.propose` call for the leader-owned view and exit that /// view via `Vote::Nullify` instead of retrying proposals through the live /// run loop. fn nullify_after_crash_in_propose_window(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"nullify_after_crash_in_propose_window".to_vec(); let partition = "nullify_after_crash_in_propose_window".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Pre-crash: drop every propose response. The leader calls // `automaton.propose`, the mock swallows the request, and nothing // is journaled. An observer records that the pre-crash leader // actually got as far as requesting a proposal so the test knows // the abort happens inside the propose window rather than before // the voter even became leader. let pre_propose_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let pre_propose_tracker = pre_propose_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); // Stall (not drop) so the voter's receiver stays open indefinitely. // Dropping the sender would fire `MissingProposal` and journal a // `Nullify` before we can abort, which would in turn cause replay // to skip the propose path entirely post-restart. app_actor.set_stall_proposals(true); app_actor.set_propose_observer(Box::new(move |ctx| { pre_propose_tracker.lock().push(ctx.view()); })); app_actor.start(); // Build and start the pre-crash voter. `leader_timeout` is long // enough that the voter won't auto-nullify before we abort, // guaranteeing the journal contains no `Nullify` either. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(600), certification_timeout: Duration::from_secs(600), timeout_retry: Duration::from_secs(600), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance into the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Wait for the pre-crash voter to call `automaton.propose` for // the leader-owned view. The observer fires before the mock parks // the response sender, so seeing this entry confirms the voter // entered the propose window and is now blocked on a response // that will never arrive. Driving the runtime forward with a // short `context.sleep` lets the voter and application tasks // progress to their next await points without consuming batcher // messages we still need for later assertions. for _ in 0..100 { if pre_propose_calls.lock().contains(&target_view) { break; } context.sleep(Duration::from_millis(10)).await; } assert!( pre_propose_calls.lock().contains(&target_view), "pre-crash voter must reach the propose window for the leader-owned view" ); // Crash: abort the voter. Because `propose` never returned, no // `Notarize` (or any other artifact for the target view) reached // the journal. handle.abort(); // Post-restart: install a fresh application that also drops // `propose` responses, as an automaton whose build fails or whose // parent fetch is abandoned does, forcing the voter to nullify the // view. A propose observer on this application is the assertion // anchor: it must record exactly one call for the target view. let post_propose_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let post_propose_tracker = post_propose_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app_restarted"), app_cfg); app_actor.set_drop_proposals(true); app_actor.set_propose_observer(Box::new(move |ctx| { post_propose_tracker.lock().push(ctx.view()); })); app_actor.start(); // Build and start the post-restart voter on the same partition // with a short `leader_timeout` so the nullify path fires promptly // once the restarted voter has had a chance to issue its single // (dropped) propose request. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(600), timeout_retry: Duration::from_secs(600), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for replay to complete and confirm we re-entered the // leader-owned target view. Journal replay saw no notarize for // this view, so the slot starts empty and the voter will call // `automaton.propose` from scratch. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { assert_eq!(current, target_view); assert_eq!(leader, Participant::new(0)); break; } batcher::Message::Constructed(_) => {} } } // Wait for the leader-timeout nullify. This also proves the // run loop stayed responsive after the dropped propose request: // the voter did not livelock trying to re-propose, it reached the // timeout path and emitted the nullify vote. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { panic!( "restarted voter must not emit a new Notarize for the \ leader-owned view without a proposal" ); } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } // Assert the core restart invariant: the restarted voter issued // `automaton.propose` at most once for the target view and then // nullified instead of retrying. let proposed = post_propose_calls.lock(); let target_call_count = proposed.iter().filter(|v| **v == target_view).count(); assert_eq!( target_call_count, 1, "restarted voter must call automaton.propose exactly once for the \ leader-owned view before nullifying (observed: {proposed:?})" ); }); } #[test_traced] fn test_nullify_after_crash_in_propose_window() { nullify_after_crash_in_propose_window(bls12381_threshold_vrf::fixture::); nullify_after_crash_in_propose_window(bls12381_threshold_vrf::fixture::); nullify_after_crash_in_propose_window(bls12381_multisig::fixture::); nullify_after_crash_in_propose_window(bls12381_multisig::fixture::); nullify_after_crash_in_propose_window(ed25519::fixture); nullify_after_crash_in_propose_window(secp256r1::fixture); } /// After restart, a proposal we already voted on must not be re-verified /// when it is re-delivered to the voter (e.g. via the automaton after /// peer vote aggregation reconstructs it). /// /// Uses a view where we are a follower (so `verify_ready` does not /// short-circuit). Pre-restart the voter verifies the leader's proposal, /// emits a `Notarize` vote, and journals it. Post-restart the voter /// replays the journaled `Notarize`; the proposal must be restored as /// `Verified` so that re-delivering the proposal does not trigger another /// `automaton.verify` call. fn no_self_verify_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"no_self_verify_after_restart".to_vec(); let partition = "no_self_verify_after_restart".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; // RoundRobin with epoch=333, n=5: view 3 -> leader=Participant::new(1). // We are Participant::new(0), so view 3 is a follower view. let target_view = View::new(3); let target_leader_idx = 1usize; let me = participants[0].clone(); let leader_pk = participants[target_leader_idx].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Pre-restart: plain application (no observers) so the voter can verify // the leader's proposal and journal its own notarize vote for view 3. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); // Build and start the pre-restart voter. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_secs(1), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance to view 3 via a synthetic finalization for // view 2. `advance_to_view` uses Sha256::hash(&[prev_view.to_be_bytes()]) as the // parent payload, which we reuse below. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Simulate the leader's proposal: broadcast its payload contents to the // application via the relay, then deliver the proposal to the voter. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"follower_proposal"]), ); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader_pk, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Wait for our local notarize (journaled) so replay has something to restore. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { assert_eq!(notarize.proposal, proposal); break; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Restart: abort the voter and construct a fresh application with // propose + verify observers to catch any spurious work for the // follower view that has a journaled local notarize vote. handle.abort(); let propose_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let verify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let propose_tracker = propose_calls.clone(); let verify_tracker = verify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (mut app_actor, application) = mocks::application::Application::new( context.child("app_restarted"), app_cfg, ); app_actor .set_propose_observer(Box::new(move |ctx| propose_tracker.lock().push(ctx.view()))); app_actor.set_verify_observer(Box::new(move |ctx, _| { verify_tracker.lock().push(ctx.view()) })); app_actor.start(); // Build and start the post-restart voter against the same journal partition. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_millis(500), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_secs(1), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for replay to complete; confirm we re-entered the follower view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { assert_eq!(current, target_view); assert_eq!(leader, Participant::from_usize(target_leader_idx)); break; } batcher::Message::Constructed(_) => {} } } // Re-deliver the proposal via the automaton (simulating peer echo after restart). // Any spurious verify for the follower view would fire on the next run-loop // iteration after the proposal is processed. We then wait for the voter to time // out (leader_timeout) and construct a Nullify for the view: by that point the // run loop has had ample opportunity to request verification, and emitting a // Nullify proves the voter reached the timeout path without ever advancing. mailbox.proposal(proposal.clone()); loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Assert the restart invariant: verify does not fire for the previously-voted // follower proposal (primary) and propose does not fire for the follower view // (trivially, since we are not its leader). let proposed = propose_calls.lock(); let verified = verify_calls.lock(); assert!( !verified.contains(&target_view), "voter must not request verification for a previously-voted view after restart (observed: {verified:?})" ); assert!( !proposed.contains(&target_view), "voter must not propose for a previously-voted view after restart (observed: {proposed:?})" ); }); } #[test_traced] fn test_no_self_verify_after_restart() { no_self_verify_after_restart(bls12381_threshold_vrf::fixture::); no_self_verify_after_restart(bls12381_threshold_vrf::fixture::); no_self_verify_after_restart(bls12381_multisig::fixture::); no_self_verify_after_restart(bls12381_multisig::fixture::); no_self_verify_after_restart(ed25519::fixture); no_self_verify_after_restart(secp256r1::fixture); } /// When the voter is the leader of a view and later reconstructs a /// notarization for the proposal it built locally, it certifies that proposal /// through the automaton like any other. /// /// Certification is the durability barrier: the leader awaits its own /// proposal's persistence (via the automaton's `certify`) before casting a /// finalize vote, so it must not short-circuit. The test asserts the end-to-end /// invariant on the live path: a `Finalize` is emitted for the leader-owned /// view and the certify observer fires for that view. fn certifies_own_proposal_when_proposing(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certifies_own_proposal_when_proposing".to_vec(); let partition = "certifies_own_proposal_when_proposing".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Install a certify observer to confirm the leader certifies its own // proposal for the leader-owned view. let certify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let certify_tracker = certify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new( move |round, _| { certify_tracker.lock().push(round.view()); true }, )), }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); // Build and start the voter wired to the observing application. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance to the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Capture the leader's local notarize so we can deliver the matching // notarization back into the voter to drive certification. let proposal = loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { break notarize.proposal; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } }; let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // A finalize for the leader-owned view proves the voter certified its // own proposal. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Finalize(finalize)) if finalize.view() == target_view => { assert_eq!(finalize.proposal, proposal); break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!( "leader-owned proposal should certify instead of nullifying view {target_view}" ); } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Assert the live invariant: the certify observer fired for the // leader-owned proposal we built ourselves. let certified = certify_calls.lock(); assert!( certified.contains(&target_view), "voter must certify its own leader-built proposal through the automaton \ (the durability barrier before finalize); observed: {certified:?}" ); }); } #[test_traced] fn test_certifies_own_proposal_when_proposing() { certifies_own_proposal_when_proposing(bls12381_threshold_vrf::fixture::); certifies_own_proposal_when_proposing(bls12381_threshold_vrf::fixture::); certifies_own_proposal_when_proposing(bls12381_multisig::fixture::); certifies_own_proposal_when_proposing(bls12381_multisig::fixture::); certifies_own_proposal_when_proposing(ed25519::fixture); certifies_own_proposal_when_proposing(secp256r1::fixture); } /// Restart analogue of `certifies_own_proposal_when_proposing`: after the voter /// has proposed and journaled a local notarize as leader, restarting recovers /// that local proposal evidence and still certifies through the automaton once /// the corresponding notarization is resolved. /// /// The replayed local notarize is what distinguishes this case from merely /// observing a leader-owned proposal certificate during catch-up. fn certifies_own_proposal_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certifies_own_proposal_after_restart".to_vec(); let partition = "certifies_own_proposal_after_restart".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Pre-restart: plain application (no observers) so the voter can // cleanly propose and journal its own notarize vote for view 2. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); // Build and start the pre-restart voter. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance to the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Wait for the voter to emit and journal its own notarize for the // leader-owned view. The captured proposal is reused post-restart // to drive certification. let proposal = loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { break notarize.proposal; } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } }; // Restart: abort the voter and construct a fresh application with a // certify observer to catch any spurious certify call for the // leader-owned view post-replay. handle.abort(); let certify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let certify_tracker = certify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new( move |round, _| { certify_tracker.lock().push(round.view()); true }, )), }; let (app_actor, application) = mocks::application::Application::new( context.child("app_restarted"), app_cfg, ); app_actor.start(); // Build and start the post-restart voter against the same journal partition. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for replay to complete; confirm we re-entered the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, leader, .. } => { assert_eq!(current, target_view); assert_eq!(leader, Participant::new(0)); break; } batcher::Message::Constructed(_) => {} } } // Deliver the matching notarization to drive certification on the // restarted voter. let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // A finalize for the leader-owned view proves replay restored the // local proposal state and the voter certified it through the automaton. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Finalize(finalize)) if finalize.view() == target_view => { assert_eq!(finalize.proposal, proposal); break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!( "leader-owned recovered proposal should certify instead of nullifying view {target_view}" ); } batcher::Message::Update { .. } => {}, batcher::Message::Constructed(_) => {} } } // Assert the restart invariant: certify fired for the leader-owned // view whose journaled local notarize replay restored the local // proposal evidence. let certified = certify_calls.lock(); assert!( certified.contains(&target_view), "voter must certify its own leader-built proposal after restart (observed: {certified:?})" ); }); } #[test_traced] fn test_certifies_own_proposal_after_restart() { certifies_own_proposal_after_restart(bls12381_threshold_vrf::fixture::); certifies_own_proposal_after_restart(bls12381_threshold_vrf::fixture::); certifies_own_proposal_after_restart(bls12381_multisig::fixture::); certifies_own_proposal_after_restart(bls12381_multisig::fixture::); certifies_own_proposal_after_restart(ed25519::fixture); certifies_own_proposal_after_restart(secp256r1::fixture); } /// Regression: when an elected leader receives an external notarization /// for a proposal it did *not* build locally, it must invoke /// `automaton.certify` before finalizing the view. The voter certifies every /// notarized proposal through the automaton, including externally-recovered /// proposals on a leader-owned view. fn certify_observer_fires_for_external_leader_proposal(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certify_observer_fires_for_external_leader_proposal".to_vec(); let partition = "certify_observer_fires_for_external_leader_proposal".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { // Set up the simulated network. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // RoundRobin with epoch=333, n=5: view 2 -> leader=Participant::new(0) = us. let target_view = View::new(2); let target_epoch = Epoch::new(333); let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); // Stall the propose response so the slot is never populated with // a locally-built proposal. The slot stays empty (proposal=None, // status=None) while the voter's internal flag `requested_build` // is true, exactly the state in which an externally-recovered // proposal lands at the leader. // // The certify observer records every `automaton.certify` call so the // final assertion can confirm certification ran for the recovered // leader-owned proposal. let certify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let certify_tracker = certify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new(move |round, _| { certify_tracker.lock().push(round.view()); true })), }; let (mut app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.set_stall_proposals(true); app_actor.start(); // Build and start the voter. Use long `leader_timeout` so the // stalled proposal does not trigger a nullify before the // conflicting notarization reaches the voter. let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch: target_epoch, floor: Floor::Genesis(mocks::application::genesis::(target_epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(600), certification_timeout: Duration::from_secs(600), timeout_retry: Duration::from_secs(600), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, _) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for startup, then advance into the leader-owned view. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { .. } => { break; } batcher::Message::Constructed(_) => {} } } advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Craft a proposal the voter could not have built locally // (distinct payload) and build its notarization from all validator // schemes. The notarization is well-formed; quorum-worth of signers // cover the proposal so it will pass `add_notarization`. let foreign_payload = Sha256::hash(&[b"foreign_leader_owned_proposal"]); let foreign_proposal = Proposal::new( Round::new(target_epoch, target_view), target_view.previous().unwrap_or(View::zero()), foreign_payload, ); let (_, foreign_notarization) = build_notarization(&schemes, &foreign_proposal, quorum); // Deliver the foreign notarization. This seeds the voter's slot // with a proposal it never built. mailbox.recovered(Certificate::Notarization(foreign_notarization)); // Wait for a `Finalize` on the leader-owned view. Observing // finalize proves the certify callback both fired and resolved // successfully. Any `Nullify` here would mean the voter never // reached the certification branch. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Finalize(finalize)) if finalize.view() == target_view => { assert_eq!(finalize.proposal, foreign_proposal); break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!( "leader-owned view with an externally-recovered proposal \ must certify via the automaton instead of nullifying \ view {target_view}" ); } batcher::Message::Update { .. } => {} batcher::Message::Constructed(_) => {} } } // The certify callback must fire for the leader-owned view: the voter // certifies its own proposals through the automaton. let certified = certify_calls.lock(); assert!( certified.contains(&target_view), "voter must invoke automaton.certify for an externally-recovered \ leader-owned proposal (observed: {certified:?})" ); }); } #[test_traced] fn test_certify_observer_fires_for_external_leader_proposal() { certify_observer_fires_for_external_leader_proposal( bls12381_threshold_vrf::fixture::, ); certify_observer_fires_for_external_leader_proposal( bls12381_threshold_vrf::fixture::, ); certify_observer_fires_for_external_leader_proposal(bls12381_multisig::fixture::); certify_observer_fires_for_external_leader_proposal( bls12381_multisig::fixture::, ); certify_observer_fires_for_external_leader_proposal(ed25519::fixture); certify_observer_fires_for_external_leader_proposal(secp256r1::fixture); } /// Test that in-flight certification requests are cancelled when finalization occurs. /// /// 1. Use a very long certify latency to ensure certification is in-flight. /// 2. Send a notarization to trigger certification. /// 3. Send a finalization for the same view before certification completes. /// 4. Verify that no Certified message is sent to the resolver. fn certification_cancelled_on_finalization(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let reporter_config = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_config); let elector = elector.build(schemes[0].participants()); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (2_000.0, 0.0), // 2 seconds should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("application"), application_cfg); actor.start(); let cfg = Config { scheme: schemes[0].clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "cert_cancel_test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (actor, mut mailbox) = Actor::new(context.child("actor"), cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(10)); let resolver = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(1024)); let batcher = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); actor.start(batcher, resolver, vote_sender, certificate_sender); // Wait for initial batcher notification if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Send a notarization for view 5 to trigger certification let view5 = View::new(5); let digest5 = Sha256::hash(&[b"payload_to_certify"]); let proposal5 = Proposal::new(Round::new(Epoch::new(333), view5), View::new(0), digest5); // Broadcast payload let contents = (proposal5.round, Sha256::hash(&[b"genesis"]), 42u64).encode(); relay.broadcast(&me, Recipients::All, (digest5, contents)); // Send proposal to verify mailbox.proposal(proposal5.clone()); // Send notarization let (_, notarization) = build_notarization(&schemes, &proposal5, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait for certification to start (it will be slow due to latency) context.sleep(Duration::from_millis(100)).await; // Send finalization for view 5 before certification completes let (_, finalization) = build_finalization(&schemes, &proposal5, quorum); mailbox.recovered(Certificate::Finalization(finalization)); // Wait for finalization to be processed loop { if let batcher::Message::Update { finalized, .. } = batcher_receiver.recv().await.unwrap() && finalized >= view5 { break; } } // Wait for resolver finalization message (skip other certificates) loop { let msg = resolver_receiver .recv() .await .expect("expected resolver msg"); match msg { MailboxMessage::Updated { certificate: Certificate::Finalization(f), .. } => { assert_eq!(f.view(), view5); break; } MailboxMessage::Updated { .. } => continue, MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => continue, MailboxMessage::Certified { .. } => { panic!("unexpected Certified message before finalization processed") } } } // Wait longer than certify_latency (2s) to verify certification was cancelled. // If certification wasn't cancelled, it would complete and send a Certified message. let certified_received = select! { msg = resolver_receiver.recv() => { matches!(msg, Some(MailboxMessage::Certified { .. })) }, _ = context.sleep(Duration::from_secs(4)) => false, }; assert!( !certified_received, "Certified message should NOT have been sent - certification should be cancelled" ); }); } #[test_traced] fn test_certification_cancelled_on_finalization() { certification_cancelled_on_finalization::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certification_cancelled_on_finalization::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certification_cancelled_on_finalization::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certification_cancelled_on_finalization::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certification_cancelled_on_finalization::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); certification_cancelled_on_finalization::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } #[test_traced] fn test_stale_same_parent_verification_failure_does_not_nullify_replacement() { let n = 5; let quorum = quorum(n); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, b"stale-same-parent-verification", n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let verify_requests = Arc::new(Mutex::new(Vec::new())); let (mut mailbox, mut batcher_receiver, mut resolver_receiver, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterOptions { leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), verify_latency_ms: 100.0, certify_latency_ms: 500.0, verify_requests: Some(verify_requests.clone()), fail_verification: true, ..Default::default() }, ) .await; if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} let view = View::new(5); let parent_payload = advance_to_view(&mut mailbox, &mut batcher_receiver, &schemes, quorum, view).await; let proposal_a = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal-a"]), ); relay.broadcast( &participants[1], Recipients::All, ( proposal_a.payload, (proposal_a.round, parent_payload, 1u64).encode(), ), ); mailbox.proposal(proposal_a); context.sleep(Duration::from_millis(10)).await; assert_eq!(*verify_requests.lock(), vec![view]); let proposal_b = Proposal::new( Round::new(Epoch::new(333), view), view.previous().unwrap(), Sha256::hash(&[b"proposal-b"]), ); let (_, notarization) = build_notarization(&schemes, &proposal_b, quorum); mailbox.recovered(Certificate::Notarization(notarization)); loop { select! { message = batcher_receiver.recv() => match message.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == view => { panic!("stale verification failure nullified the replacement proposal"); } batcher::Message::Update { .. } | batcher::Message::Constructed(_) => {} }, message = resolver_receiver.recv() => match message.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == view => { assert!(success); break; } MailboxMessage::Updated { .. } | MailboxMessage::Certified { .. } | MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => {} }, _ = context.sleep(Duration::from_secs(3)) => { panic!("replacement proposal did not finish certification"); }, } } }); } /// Test that in-flight certification is still reported to resolver after nullification. /// /// 1. Use a long certify latency so certification remains in-flight. /// 2. Send notarization to trigger certification. /// 3. Send nullification for the same view before certification completes. /// 4. Verify that a Certified message is still sent to resolver when certification completes. fn certification_still_reports_to_resolver_after_nullification( mut fixture: F, elector: L, ) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let reporter_config = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_config); let elector = elector.build(schemes[0].participants()); let relay = Arc::new(mocks::relay::Relay::::new()); let application_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (2_000.0, 0.0), // 2 seconds should_certify: mocks::application::Certifier::Always, }; let (actor, application) = mocks::application::Application::new(context.child("application"), application_cfg); actor.start(); let cfg = Config { scheme: schemes[0].clone(), elector, blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: "cert_after_nullification_test".to_string(), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (actor, mut mailbox) = Actor::new(context.child("actor"), cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(10)); let resolver = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(1024)); let batcher = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); actor.start(batcher, resolver, vote_sender, certificate_sender); // Wait for initial batcher notification if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Send a notarization for view 5 to trigger certification let view5 = View::new(5); let digest5 = Sha256::hash(&[b"payload_to_certify"]); let proposal5 = Proposal::new(Round::new(Epoch::new(333), view5), View::new(0), digest5); // Broadcast payload let contents = (proposal5.round, Sha256::hash(&[b"genesis"]), 42u64).encode(); relay.broadcast(&me, Recipients::All, (digest5, contents)); // Send proposal and notarization mailbox.proposal(proposal5.clone()); let (_, notarization) = build_notarization(&schemes, &proposal5, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait for certification to start (it will be slow due to latency) context.sleep(Duration::from_millis(100)).await; // Send nullification for the same view before certification completes let (_, nullification) = build_nullification(&schemes, Round::new(Epoch::new(333), view5), quorum); mailbox.recovered(Certificate::Nullification(nullification)); // Even after nullification, late certification should still be forwarded to resolver. let reported = loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == view5 => { break Some(success); } MailboxMessage::Certified { .. } | MailboxMessage::Updated { .. } | MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { .. } = msg.unwrap() {} }, _ = context.sleep(Duration::from_secs(6)) => { break None; }, } }; assert_eq!( reported, Some(true), "expected resolver to receive successful certification after nullification" ); }); } #[test_traced] fn test_certification_still_reports_to_resolver_after_nullification() { certification_still_reports_to_resolver_after_nullification::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certification_still_reports_to_resolver_after_nullification::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); certification_still_reports_to_resolver_after_nullification::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certification_still_reports_to_resolver_after_nullification::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); certification_still_reports_to_resolver_after_nullification::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); certification_still_reports_to_resolver_after_nullification::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Regression: a notarization arriving after nullification for the same view /// should still trigger certification. fn late_notarization_after_nullification_still_certifies(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"late_notarization_after_nullification".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { // Create simulated network. // Build participants and voter. let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let (mut mailbox, mut batcher_receiver, mut resolver_receiver, _, _) = setup_voter( &context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterOptions { leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_secs(5), ..Default::default() }, ) .await; // Move into a concrete current view. let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Nullify current view first. let (_, nullification) = build_nullification(&schemes, Round::new(Epoch::new(333), target_view), quorum); mailbox.recovered(Certificate::Nullification(nullification)); // Then provide notarization for that same view. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"late_notarization_after_nullification"]), ); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); let certified = loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { break Some(success); } MailboxMessage::Certified { .. } | MailboxMessage::Updated { .. } | MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { .. } = msg.unwrap() {} }, _ = context.sleep(Duration::from_secs(6)) => break None, } }; assert_eq!( certified, Some(true), "expected notarization after nullification to still trigger certification" ); }); } #[test_traced] fn test_late_notarization_after_nullification_still_certifies() { late_notarization_after_nullification_still_certifies::<_, _>( bls12381_threshold_vrf::fixture::, ); late_notarization_after_nullification_still_certifies::<_, _>( bls12381_threshold_vrf::fixture::, ); late_notarization_after_nullification_still_certifies::<_, _>( bls12381_multisig::fixture::, ); late_notarization_after_nullification_still_certifies::<_, _>( bls12381_multisig::fixture::, ); late_notarization_after_nullification_still_certifies::<_, _>(ed25519::fixture); late_notarization_after_nullification_still_certifies::<_, _>(secp256r1::fixture); } /// Tests certification after: timeout -> receive notarization -> certify. /// This test does NOT send a notarize vote first (we timeout before receiving a proposal). fn certification_after_timeout(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certification_after_timeout".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let elector = RoundRobin::::default(); let built_elector: RoundRobinElector = elector .clone() .build(&participants.clone().try_into().unwrap()); let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(100), ..Default::default() }, ) .await; // Advance to view 3 where we're a follower. // With RoundRobin, epoch=333, n=5: leader = (333 + view) % 5 // View 3: leader = 1 (not us) let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; assert_ne!( built_elector.elect(Round::new(Epoch::new(333), target_view), ()), Participant::new(0), "we should not be leader at view 3" ); // Wait for timeout (nullify vote) WITHOUT sending notarize first loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(n)) if n.view() == target_view => break, batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(15)) => { panic!("expected nullify vote"); }, } } // Send notarization certificate (simulating delayed network delivery) let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"timeout_test"]), ); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Verify view advances let advanced = loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > target_view { break true; } }, _ = context.sleep(Duration::from_secs(5)) => { break false; }, } }; assert!( advanced, "view should advance after certification (timeout case)" ); }); } #[test_traced] fn test_certification_after_timeout() { certification_after_timeout::<_, _>(bls12381_threshold_vrf::fixture::); certification_after_timeout::<_, _>(bls12381_threshold_vrf::fixture::); certification_after_timeout::<_, _>(bls12381_multisig::fixture::); certification_after_timeout::<_, _>(bls12381_multisig::fixture::); certification_after_timeout::<_, _>(ed25519::fixture); certification_after_timeout::<_, _>(secp256r1::fixture); } /// Tests certification after: notarize -> timeout -> receive notarization -> certify. /// This test runs when we are NOT the leader (receiving proposal from another participant). fn certification_after_notarize_timeout_as_follower(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certification_after_notarize_timeout_as_follower".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let elector = RoundRobin::::default(); let built_elector: RoundRobinElector = elector .clone() .build(&participants.clone().try_into().unwrap()); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(100), ..Default::default() }, ) .await; // Advance to view 3 where we're a follower. // With RoundRobin, epoch=333, n=5: leader = (333 + view) % 5 // View 3: leader = 1 (not us) let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; assert_ne!( built_elector.elect(Round::new(Epoch::new(333), target_view), ()), Participant::new(0), "we should not be leader at view 3" ); // Create and send proposal as if from the leader (participant 1) let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"follower_test"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Wait for notarize vote loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(n)) if n.view() == target_view => break, batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected notarize vote"); }, } } // Trigger timeout context.sleep(Duration::from_secs(11)).await; // Wait for nullify vote loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(n)) if n.view() == target_view => break, batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!("expected nullify vote"); }, } } // Send notarization certificate let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Verify view advances let advanced = loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > target_view { break true; } }, _ = context.sleep(Duration::from_secs(5)) => { break false; }, } }; assert!( advanced, "view should advance after certification (follower case)" ); }); } #[test_traced] fn test_certification_after_notarize_timeout_as_follower() { certification_after_notarize_timeout_as_follower::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_after_notarize_timeout_as_follower::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_after_notarize_timeout_as_follower::<_, _>( bls12381_multisig::fixture::, ); certification_after_notarize_timeout_as_follower::<_, _>( bls12381_multisig::fixture::, ); certification_after_notarize_timeout_as_follower::<_, _>(ed25519::fixture); certification_after_notarize_timeout_as_follower::<_, _>(secp256r1::fixture); } /// Tests certification after: notarize -> timeout -> receive notarization -> certify. /// This test runs when we ARE the leader (proposing ourselves). fn certification_after_notarize_timeout_as_leader(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"certification_after_notarize_timeout_as_leader".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup application mock and voter let elector = RoundRobin::::default(); let built_elector: RoundRobinElector = elector .clone() .build(&participants.clone().try_into().unwrap()); let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(10), certification_timeout: Duration::from_secs(10), timeout_retry: Duration::from_secs(100), ..Default::default() }, ) .await; // Advance to view 2 where we ARE the leader. // With RoundRobin, epoch=333, n=5: leader = (333 + view) % 5 // View 2: leader = 0 (us) let target_view = View::new(2); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; assert_eq!( built_elector.elect(Round::new(Epoch::new(333), target_view), ()), Participant::new(0), "we should be leader at view 2" ); // As leader, wait for our own notarize vote (automaton will propose) let proposal = loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(n)) if n.view() == target_view => { break n.proposal.clone(); } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected notarize vote as leader"); }, } }; // Trigger timeout context.sleep(Duration::from_secs(11)).await; // Wait for nullify vote loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(n)) if n.view() == target_view => break, batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(1)) => { panic!("expected nullify vote"); }, } } // Send notarization certificate (as if other participants formed it) let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Verify view advances let advanced = loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > target_view { break true; } }, _ = context.sleep(Duration::from_secs(5)) => { break false; }, } }; assert!( advanced, "view should advance after certification (leader case)" ); }); } #[test_traced] fn test_certification_after_notarize_timeout_as_leader() { certification_after_notarize_timeout_as_leader::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_after_notarize_timeout_as_leader::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_after_notarize_timeout_as_leader::<_, _>( bls12381_multisig::fixture::, ); certification_after_notarize_timeout_as_leader::<_, _>( bls12381_multisig::fixture::, ); certification_after_notarize_timeout_as_leader::<_, _>(ed25519::fixture); certification_after_notarize_timeout_as_leader::<_, _>(secp256r1::fixture); } /// Tests that when certification returns a cancelled receiver, the voter doesn't hang /// and continues to make progress (via voting to nullify the view that could not be certified). fn cancelled_certification_does_not_hang(mut fixture: F, traces: TraceStorage) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default(); // Set up voter with Certifier::Cancel let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { certification_timeout: Duration::from_millis(500), timeout_retry: Duration::from_mins(60), certifier: mocks::application::Certifier::Cancel, ..Default::default() }, ) .await; // Advance to view 3 where we're a follower. // With RoundRobin, epoch=333, n=5: leader = (333 + view) % 5 // View 3: leader = 1 (not us) let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Broadcast the payload contents so verification can complete. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"test_proposal"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay .broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Build and send notarization so the voter tries to certify let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // Certification will be cancelled, so the voter should eventually timeout // and emit a nullify vote. loop { select! { msg = batcher_receiver.recv() => { match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} } }, _ = context.sleep(Duration::from_secs(5)) => { panic!( "voter should emit nullify for view {target_view} despite cancelled certification", ); }, } } // Verify the "failed to certify proposal" log was emitted with the correct round let expected_round = format!("Round {{ epoch: Epoch(333), view: View({target_view}) }}"); traces .get_by_level(Level::DEBUG) .expect_event(|event| { event.metadata.content == "failed to certify proposal" && event .metadata .fields .iter() .any(|(name, value)| name == "err" && value == "RecvError(())") && event .metadata .fields .iter() .any(|(name, value)| name == "round" && value == &expected_round) }) .unwrap(); }); } #[test_collect_traces] fn test_cancelled_certification_does_not_hang(traces: TraceStorage) { cancelled_certification_does_not_hang( bls12381_threshold_vrf::fixture::, traces.clone(), ); cancelled_certification_does_not_hang( bls12381_threshold_vrf::fixture::, traces.clone(), ); cancelled_certification_does_not_hang( bls12381_multisig::fixture::, traces.clone(), ); cancelled_certification_does_not_hang( bls12381_multisig::fixture::, traces.clone(), ); cancelled_certification_does_not_hang(ed25519::fixture, traces.clone()); cancelled_certification_does_not_hang(secp256r1::fixture, traces); } /// Verifies that per-view work is attributed to view-scoped spans: /// 1. The automaton propose request nests under `simplex.voter.view`. /// 2. Vote broadcast (within notify) nests under `simplex.voter.view`. /// 3. Mailbox messages carry their enqueue span into voter processing. fn view_spans_track_automaton_boundary(mut fixture: F, traces: TraceStorage) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"consensus".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default(); let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), ..Default::default() }, ) .await; // Advance to view 2, where we are the leader. // With RoundRobin, epoch=333, n=5: leader = (333 + view) % 5 let target_view = View::new(2); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Wait for the notarize vote on our own proposal. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => { break; } _ => {} } } // The propose request to the automaton runs under the view span. traces .get_by_level(Level::DEBUG) .expect_event(|event| { event.metadata.content == "requested proposal from automaton" && event .expect_span_at_index(0, |span| { span.expect_content_exact("simplex.voter.propose")?; // Operation spans carry both epoch and view so they // are unambiguous when viewed standalone. span.expect_field_exact("epoch", "333")?; span.expect_field_exact("view", "2") }) .is_ok() && event .expect_span(|span| { span.content == "simplex.voter.view" && span.expect_field_exact("epoch", "333").is_ok() && span.expect_field_exact("view", "2").is_ok() }) .is_ok() }) .unwrap(); // The notarize broadcast happens within notify under the view span. traces .get_by_level(Level::DEBUG) .expect_event(|event| { event.metadata.content == "broadcasting notarize" && event .expect_span_at_index(0, |span| { span.expect_content_exact("simplex.voter.notify") }) .is_ok() && event .expect_span(|span| span.content == "simplex.voter.view") .is_ok() }) .unwrap(); // Mailbox messages carry their enqueue span into voter processing. traces .get_by_level(Level::TRACE) .expect_event(|event| { event.metadata.content == "received finalization" && event .expect_span_at_index(0, |span| { span.expect_content_exact("simplex.voter.process") }) .is_ok() && event .expect_span(|span| span.content == "simplex.voter.mailbox.recovered") .is_ok() }) .unwrap(); }); } #[test_collect_traces] fn test_view_spans_track_automaton_boundary(traces: TraceStorage) { view_spans_track_automaton_boundary(ed25519::fixture, traces); } /// Regression: a canceled certification attempt must not be persisted as failure. /// /// We first trigger a canceled certify receiver, restart the voter, and then require: /// 1. successful certification for the same view from replayed notarization state, and /// 2. no immediate timeout/nullify for that view after restart. fn cancelled_certification_recertifies_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"cancelled_cert_restart_recertify".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let partition = "cancelled_certification_recertifies_after_restart".to_string(); let epoch = Epoch::new(333); // First run: certification receiver gets cancelled. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Cancel, }; let (app_actor, application) = mocks::application::Application::new(context.child("app_cancel"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter_cancel"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() { } let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap(), Sha256::hash(&[b"restart_recertify_payload"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // Give the canceled certification attempt time to run before restart. context.sleep(Duration::from_millis(200)).await; // Sanity check: canceled certification should not have advanced this view yet. let advanced_before_restart = select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() { current > target_view } else { false } }, _ = context.sleep(Duration::from_millis(200)) => false, }; assert!( !advanced_before_restart, "view should not advance before restart when certification receiver is canceled" ); // Restart voter. handle.abort(); // Second run: certification should succeed from replayed state. // Use a longer certify latency so there is a real window where an // incorrect immediate nullify could fire after restart. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (2_000.0, 0.0), // 2 seconds should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app_restarted"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() { } loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { assert!(success, "expected successful certification after restart for canceled certification view"); break; } MailboxMessage::Certified { .. } | MailboxMessage::Updated { .. } | MailboxMessage::Resolve { .. } | MailboxMessage::Replayed { .. } => {} }, msg = batcher_receiver.recv() => { match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!("unexpected immediate nullify for view {target_view} after restart"); } batcher::Message::Update { .. } => { } _ => {} } }, _ = context.sleep(Duration::from_secs(5)) => { panic!( "timed out waiting for successful certification for restarted view {target_view}" ); }, } }; // Give reporter a moment to ingest any late events and ensure no nullify artifacts // were emitted for the restarted target view. context.sleep(Duration::from_millis(100)).await; assert!( !reporter.nullifies.lock().contains_key(&target_view), "did not expect nullify votes for restarted view {target_view}" ); assert!( !reporter.nullifications.lock().contains_key(&target_view), "did not expect nullification certificate for restarted view {target_view}" ); }); } #[test_traced] fn test_cancelled_certification_recertifies_after_restart() { cancelled_certification_recertifies_after_restart::<_, _>( bls12381_threshold_vrf::fixture::, ); cancelled_certification_recertifies_after_restart::<_, _>( bls12381_threshold_vrf::fixture::, ); cancelled_certification_recertifies_after_restart::<_, _>( bls12381_multisig::fixture::, ); cancelled_certification_recertifies_after_restart::<_, _>( bls12381_multisig::fixture::, ); cancelled_certification_recertifies_after_restart::<_, _>(ed25519::fixture); cancelled_certification_recertifies_after_restart::<_, _>(secp256r1::fixture); } /// Demonstrates that validators in future views cannot retroactively help /// stuck validators escape via nullification. /// /// This test extends the previous scenario to show that: /// 1. A stuck validator (view 3) cannot be rescued by notarizations from future views /// 2. The only escape route is a finalization certificate (which requires Byzantine cooperation) /// /// Once the f+1 honest validators certify view 3 and advance to view 4, /// they can only vote to nullify view 4 (their current view) without equivocating. /// The `timeout` function only votes to nullify `self.view` (current view). fn only_finalization_rescues_validator(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 4; let quorum = quorum(n); let namespace = b"future_notarization_no_rescue".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(60)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; // Setup voter with Certifier::Cancel to simulate missing verification context. let elector = RoundRobin::::default(); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector.clone(), VoterOptions { leader_timeout: Duration::from_secs(2), certification_timeout: Duration::from_secs(3), timeout_retry: Duration::from_secs(1), certifier: mocks::application::Certifier::Cancel, ..Default::default() }, ) .await; // Advance to view 4 so the stuck round is not leader-owned by this validator. let view_4 = View::new(4); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, view_4, ) .await; let proposal_4 = Proposal::new( Round::new(Epoch::new(333), view_4), view_4.previous().unwrap(), Sha256::hash(&[b"view_4_proposal"]), ); let leader = participants[1].clone(); let contents = (proposal_4.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal_4.payload, contents)); mailbox.proposal(proposal_4.clone()); let (_, notarization_4) = build_notarization(&schemes, &proposal_4, quorum); mailbox.recovered(Certificate::Notarization(notarization_4)); // Wait for the first nullify vote (confirms stuck state) loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(n)) if n.view() == view_4 => break, batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(10)) => { panic!("expected nullify vote for view 4"); }, } } // Now simulate what the "advanced" validators (f+1 honest with context) are doing: // They certified view 4 and advanced to view 5, where they're making progress. // Send a notarization for view 5 to the stuck validator. let view_5 = View::new(5); let proposal_5 = Proposal::new( Round::new(Epoch::new(333), view_5), view_4, // Parent is view 4 (certified by the advanced validators) Sha256::hash(&[b"view_5_proposal"]), ); let (_, notarization_5) = build_notarization(&schemes, &proposal_5, quorum); // Send the view 5 notarization to the stuck validator mailbox.recovered(Certificate::Notarization(notarization_5)); // The stuck validator should still not advance. // // Receiving a notarization for view 5 doesn't help because: // 1. add_notarization() does not call enter_view() - it only adds to certification_candidates // 2. To advance past view 4, the validator needs EITHER: // a. Certification of view 4 to succeed (impossible - no context) // b. A nullification certificate for view 4 (impossible - only f votes) // c. A finalization certificate (requires Byzantine to vote finalize) // Use a fixed deadline: timeout retries re-send the nullify vote every // second, which would otherwise restart a per-iteration sleep forever. let deadline = context.current() + Duration::from_secs(5); let advanced = loop { select! { msg = batcher_receiver.recv() => { match msg.unwrap() { batcher::Message::Update { current, .. } if current > view_4 => { break true; } batcher::Message::Constructed(Vote::Nullify(n)) => { // Still voting nullify for view 4 - expected assert_eq!( n.view(), view_4, "should only vote nullify for stuck view" ); } _ => {} } }, _ = context.sleep_until(deadline) => { break false; }, } }; assert!( !advanced, "receiving a notarization for view 5 should NOT rescue the stuck validator - \ they still can't certify view 4 (no context) and can't form a nullification \ (not enough votes). The f+1 honest validators who advanced to view 5 cannot \ retroactively help because they can only vote nullify for their current view (5), \ not for view 4." ); // HOWEVER: A finalization certificate WOULD rescue the stuck validator. // If the Byzantine validators eventually cooperate and vote finalize, // the finalization would abort the stuck certification and advance the view. // // Let's demonstrate this escape route works (if Byzantine cooperate): let (_, finalization_5) = build_finalization(&schemes, &proposal_5, quorum); mailbox.recovered(Certificate::Finalization(finalization_5)); // Now the validator SHOULD advance (finalization aborts stuck certification) let deadline = context.current() + Duration::from_secs(5); let rescued = loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > view_5 { break true; } }, _ = context.sleep_until(deadline) => { break false; }, } }; assert!( rescued, "a finalization certificate SHOULD rescue the stuck validator - \ this is the ONLY escape route, but it requires Byzantine cooperation \ (they must vote finalize). If Byzantine permanently withhold finalize votes, \ the stuck validators are permanently excluded from consensus." ); }); } #[test_traced] fn test_only_finalization_rescues_validator() { only_finalization_rescues_validator::<_, _>(bls12381_threshold_vrf::fixture::); only_finalization_rescues_validator::<_, _>(bls12381_threshold_vrf::fixture::); only_finalization_rescues_validator::<_, _>(bls12381_multisig::fixture::); only_finalization_rescues_validator::<_, _>(bls12381_multisig::fixture::); only_finalization_rescues_validator::<_, _>(ed25519::fixture); only_finalization_rescues_validator::<_, _>(secp256r1::fixture); } /// Tests that when certification explicitly fails (returns false), the voter: /// 1. Can vote nullify even after having voted notarize /// 2. Will emit a nullify vote immediately after certification failure /// /// This simulates the coding marshal scenario where: /// - verify() returns true (shard validity passes) /// - Voter votes notarize /// - Notarization forms /// - certify() returns false (block context mismatch discovered during deferred_verify) /// - Voter should vote nullify to attempt to advance /// /// The liveness concern is: if only f honest validators can vote nullify (the ones who /// never saw the shard/never verified), then nullification quorum (2f+1) cannot form /// since the f+1 honest who voted notarize need to also vote nullify. fn certification_failure_allows_nullify_after_notarize(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"cert_fail_nullify".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default(); // Set up voter with Certifier::Custom that always returns false // This simulates coding marshal's deferred_verify finding context mismatch let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { // Long timeouts to prove nullify comes from cert failure leader_timeout: Duration::from_secs(100), certification_timeout: Duration::from_secs(100), timeout_retry: Duration::from_secs(100), certifier: mocks::application::Certifier::Custom(Box::new(|_, _| false)), ..Default::default() }, ) .await; // Advance to view 3 where we're a follower. let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Broadcast the payload contents so verification can complete. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"test_proposal"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Wait for notarize vote first (verification passes) loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(n)) if n.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected notarize vote for view {target_view}"); }, } } // Build and send notarization so the voter tries to certify let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // Certification will fail (returns false), so the voter should emit a nullify vote. // This must happen quickly (not after 100s timeout) to prove it's from cert failure. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { // Successfully voted nullify after having voted notarize break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!( "voter should emit nullify for view {target_view} after certification failure, \ even though it already voted notarize" ); }, } } }); } #[test_traced] fn test_certification_failure_allows_nullify_after_notarize() { certification_failure_allows_nullify_after_notarize::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_failure_allows_nullify_after_notarize::<_, _>( bls12381_threshold_vrf::fixture::, ); certification_failure_allows_nullify_after_notarize::<_, _>( bls12381_multisig::fixture::, ); certification_failure_allows_nullify_after_notarize::<_, _>( bls12381_multisig::fixture::, ); certification_failure_allows_nullify_after_notarize::<_, _>(ed25519::fixture); certification_failure_allows_nullify_after_notarize::<_, _>(secp256r1::fixture); } /// Verify that a voter recovers via timeout when certification hangs indefinitely. /// /// This simulates the scenario where a notarization forms but the block is /// unrecoverable. This includes the restart case where the local block was not /// durable and no peer can provide enough data for reconstruction. In this /// case, `certify()` subscribes to the block /// but the subscription never resolves. The voter must rely on the view timeout /// to emit a nullify vote and advance the chain. /// /// Unlike `Cancel` mode (where the certify receiver errors immediately), `Pending` /// mode holds the certify sender alive so the future never completes, forcing the /// voter to recover purely through its timeout mechanism. fn pending_certification_nullifies_on_timeout(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"pending_cert_nullify".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(10)); executor.start(|mut context| async move { // Get participants let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let elector = RoundRobin::::default(); // Set up voter with Certifier::Pending (certify hangs indefinitely). let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(3), certification_timeout: Duration::from_secs(4), timeout_retry: Duration::from_mins(60), certifier: mocks::application::Certifier::Pending, ..Default::default() }, ) .await; // Advance to view 3 where we're a follower. let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Broadcast the payload contents so verification can complete. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"test_proposal"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Wait for notarize vote (verification passes). loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(n)) if n.view() == target_view => { break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected notarize vote for view {target_view}"); }, } } // Build and send notarization so the voter tries to certify. let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Certification hangs (sender held alive, receiver pending). The voter // must recover via the view timeout and emit a nullify vote. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { // Timeout fired and voter emitted nullify despite // certification being indefinitely pending. break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(8)) => { panic!( "voter should emit nullify for view {target_view} via timeout \ when certification hangs indefinitely", ); }, } } // The local nullify vote alone is not enough to advance. Without a // nullification or finalization certificate, the voter must remain in // the same view while certification is still pending. let advanced_without_certificate = loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Update { current, .. } if current > target_view => { break true; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { break false; }, } }; assert!( !advanced_without_certificate, "voter should not advance past view {target_view} without a nullification or finalization certificate" ); }); } #[test_traced] fn test_pending_certification_nullifies_on_timeout() { pending_certification_nullifies_on_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); pending_certification_nullifies_on_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); pending_certification_nullifies_on_timeout::<_, _>(bls12381_multisig::fixture::); pending_certification_nullifies_on_timeout::<_, _>(bls12381_multisig::fixture::); pending_certification_nullifies_on_timeout::<_, _>(ed25519::fixture); pending_certification_nullifies_on_timeout::<_, _>(secp256r1::fixture); } /// Regression: once a proposal is received, leader timeout must no longer fire for that view. /// /// We require: /// 1. No nullify before `certification_timeout` even though `leader_timeout` has elapsed. /// 2. Nullify eventually arrives only after `certification_timeout` when no /// certificate progress occurs. fn proposal_clears_leader_timeout_before_certification_timeout(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"proposal_clears_leader_timeout".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(15)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default(); let (mut mailbox, mut batcher_receiver, _, relay, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), ..Default::default() }, ) .await; // Advance to a follower view. let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Submit proposal quickly so leader timeout is cleared. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"proposal_clears_leader_timeout"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Ensure proposal verification path ran. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(v)) if v.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected notarize vote for view {target_view}"); }, } } // `leader_timeout` is 1s and `certification_timeout` is 5s. We should not // see nullify in this 2s window after proposal handling, even though // leader timeout has elapsed. let no_nullify_deadline = context.current() + Duration::from_secs(2); loop { select! { _ = context.sleep_until(no_nullify_deadline) => break, msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!( "received nullify for view {target_view} before certification timeout" ); } batcher::Message::Update { .. } => {}, _ => {} } } } // After certification timeout elapses, timeout recovery must emit nullify. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(6)) => { panic!( "expected nullify for view {target_view} after certification timeout" ); }, } } }); } #[test_traced] fn test_proposal_clears_leader_timeout_before_certification_timeout() { proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_multisig::fixture::, ); proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_multisig::fixture::, ); proposal_clears_leader_timeout_before_certification_timeout::<_, _>(ed25519::fixture); proposal_clears_leader_timeout_before_certification_timeout::<_, _>(secp256r1::fixture); } /// Regression: proposals recovered from notarization certificates must clear the /// current view's leader timeout without emitting a local notarize vote. /// /// We require: /// 1. No nullify before `certification_timeout` even though `leader_timeout` has elapsed. /// 2. Nullify eventually arrives only after `certification_timeout` when certification /// remains pending. fn recovered_proposal_clears_leader_timeout_before_certification_timeout(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"recovered_proposal_clears_leader_timeout".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(15)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let elector = RoundRobin::::default(); let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), certifier: mocks::application::Certifier::Pending, ..Default::default() }, ) .await; // Advance to a follower view. let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Recover a notarization that carries the proposal for this view. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"recovered_proposal_clears_leader_timeout"]), ); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox .recovered(Certificate::Notarization(notarization)); // `leader_timeout` is 1s and `certification_timeout` is 5s. We should not // emit a notarize vote or nullify in this 2s window after certificate handling, // even though leader timeout has elapsed. let quiet_deadline = context.current() + Duration::from_secs(2); loop { select! { _ = context.sleep_until(quiet_deadline) => break, msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(v)) if v.view() == target_view => { panic!( "unexpected notarize for view {target_view} from recovered certificate" ); } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { panic!( "received nullify for view {target_view} before certification timeout after recovered certificate" ); } batcher::Message::Update { .. } => {}, _ => {} } } } // After certification timeout elapses, timeout recovery must emit nullify. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == target_view => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(6)) => { panic!( "expected nullify for view {target_view} after certification timeout with recovered certificate" ); }, } } }); } #[test_traced] fn test_recovered_proposal_clears_leader_timeout_before_certification_timeout() { recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_threshold_vrf::fixture::, ); recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_multisig::fixture::, ); recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( bls12381_multisig::fixture::, ); recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( ed25519::fixture, ); recovered_proposal_clears_leader_timeout_before_certification_timeout::<_, _>( secp256r1::fixture, ); } /// Regression: after a timed-out view is nullified and the voter advances, /// the next view must start with a fresh leader timeout. fn next_view_gets_fresh_timeout_after_prior_view_nullifies(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"next_view_gets_fresh_timeout_after_prior_view_nullifies".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(15)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true, ) .await; let (mut mailbox, mut batcher_receiver, _, _, _) = setup_voter( &context, &oracle, &participants, &schemes, RoundRobin::::default(), VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), ..Default::default() }, ) .await; // Wait for the initial view 1 batcher update. loop { match batcher_receiver.recv().await.unwrap() { batcher::Message::Update { current, .. } => { if current == View::new(1) { break; } } batcher::Message::Constructed(_) => {} } } // Allow view 1 to time out and emit a nullify vote. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(1) => { break; } batcher::Message::Update { .. } => {}, _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected nullify for view 1"); }, } } // Deliver a nullification certificate for view 1 so the voter enters view 2. let (_, nullification) = build_nullification(&schemes, Round::new(Epoch::new(333), View::new(1)), quorum); mailbox .recovered(Certificate::Nullification(nullification)); loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(2) => { panic!( "received nullify for view 2 before its fresh leader timeout elapsed" ); } batcher::Message::Update { current, .. } if current == View::new(2) => { break; } _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected voter to advance to view 2"); }, } } // The old view timed out, but the new view should still get its own leader timeout // rather than immediately nullifying on entry. let quiet_deadline = context.current() + Duration::from_millis(500); loop { select! { _ = context.sleep_until(quiet_deadline) => break, msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(2) => { panic!( "received nullify for view 2 before its fresh leader timeout elapsed" ); } batcher::Message::Update { .. } => {}, _ => {} } } } }); } #[test_traced] fn test_next_view_gets_fresh_timeout_after_prior_view_nullifies() { next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>( bls12381_threshold_vrf::fixture::, ); next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>( bls12381_threshold_vrf::fixture::, ); next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>( bls12381_multisig::fixture::, ); next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>( bls12381_multisig::fixture::, ); next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>(ed25519::fixture); next_view_gets_fresh_timeout_after_prior_view_nullifies::<_, _>(secp256r1::fixture); } /// Regression: the first view should make progress without timing out when peers are online. /// /// We require: /// 1. No `nullify(1)` is emitted while quorum certificates arrive promptly. /// 2. The voter emits `notarize(1)`. /// 3. After successful certification, the voter emits `finalize(1)` before /// advancing to view 2. fn first_view_progress_without_timeout(mut fixture: F, elector: L) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, L: elector::Config, { let n = 5; let quorum = quorum(n); let namespace = b"first_view_progress_without_timeout".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(15)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); // Create simulated network let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let first_round = Round::new(Epoch::new(333), View::new(1)); let leader_idx = elector .clone() .build(schemes[0].participants()) .elect(first_round, >::Mode::input(None)); let leader = participants[usize::from(leader_idx)].clone(); let (mut mailbox, mut batcher_receiver, _, relay, reporter) = setup_voter( &context, &oracle, &participants, &schemes, elector, VoterOptions { leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), ..Default::default() }, ) .await; // Wait for initial batcher notification. let message = batcher_receiver.recv().await.unwrap(); match message { batcher::Message::Update { current, finalized, .. } => { assert_eq!(current, View::new(1)); assert_eq!(finalized, View::new(0)); } _ => panic!("unexpected batcher message"), } // Build a valid first-view proposal (parent is genesis at view 0). let genesis = Sha256::hash(&[&(bytes::Bytes::from_static(b"genesis"), Epoch::new(333)).encode()]); let proposal = Proposal::new( first_round, View::zero(), Sha256::hash(&[b"first_view_progress_without_timeout"]), ); let contents = (proposal.round, genesis, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // The voter should notarize view 1 and must not nullify it. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == View::new(1) => { break; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(1) => { panic!("unexpected nullify for view 1 while peers are online"); } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(2)) => { panic!("expected notarize for view 1"); }, } } // Deliver quorum notarization and ensure we finalize + advance to view 2 without nullify. let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); let deadline = context.current() + Duration::from_secs(3); let reached_view2 = loop { select! { _ = context.sleep_until(deadline) => break false, msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Finalize(finalize)) if finalize.view() == View::new(1) => { break false; } batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(1) => { panic!("unexpected nullify for view 1 while peers are online"); } batcher::Message::Update { current, .. } if current >= View::new(2) => { break true; } _ => {} }, } }; assert!(!reached_view2, "view advanced before finalize for view 1"); let reached_view2 = loop { select! { _ = context.sleep_until(deadline) => break false, msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == View::new(1) => { panic!("unexpected nullify for view 1 while peers are online"); } batcher::Message::Update { current, .. } if current >= View::new(2) => { break true; } _ => {} }, } }; assert!(reached_view2, "expected progress to view 2 from view 1"); // Give the reporter a moment to receive any late events and verify no first-view nullify artifacts. context.sleep(Duration::from_millis(50)).await; assert!( !reporter.nullifies.lock().contains_key(&View::new(1)), "did not expect nullify votes for view 1" ); assert!( !reporter.nullifications.lock().contains_key(&View::new(1)), "did not expect a nullification certificate for view 1" ); }); } #[test_traced] fn test_first_view_progress_without_timeout() { first_view_progress_without_timeout::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); first_view_progress_without_timeout::<_, _, Random>( bls12381_threshold_vrf::fixture::, Random::new(RandomVersion::V1), ); first_view_progress_without_timeout::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); first_view_progress_without_timeout::<_, _, RoundRobin>( bls12381_multisig::fixture::, RoundRobin::default(), ); first_view_progress_without_timeout::<_, _, RoundRobin>( ed25519::fixture, RoundRobin::default(), ); first_view_progress_without_timeout::<_, _, RoundRobin>( secp256r1::fixture, RoundRobin::default(), ); } /// Certification and the finalize vote share one durable section sync. /// /// 1. Gate the sync and verify the finalize reaches neither batcher nor network. /// 2. Release the sync, observe both deliveries, and abort the voter. /// 3. Restart, replay both artifacts, and advance without re-certifying. fn successful_certification_replayed_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"successful_cert_replay".to_vec(); let partition = "successful_cert_replay".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true).await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg.clone()); let relay = Arc::new(mocks::relay::Relay::::new()); let epoch = Epoch::new(333); let target_view = View::new(3); let pending_syncs = PendingSyncs::default(); let voter_context = DelayedSyncContext { inner: context.child("voter"), pending: pending_syncs.clone(), }; let pending_syncs_for_certifier = pending_syncs.clone(); // Arm the sync gate immediately before the target certification succeeds. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new( move |round, _| { assert_eq!(round.view(), target_view); pending_syncs_for_certifier.arm(); true }, )), }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(voter_context, voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let observer = participants[1].clone(); let (_, mut observer_vote_receiver) = oracle .control(observer.clone()) .register(0, TEST_QUOTA) .await .unwrap(); oracle .add_link( me.clone(), observer, Link { latency: Duration::ZERO, jitter: Duration::ZERO, success_rate: probability!(1.0), }, ) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); assert!(matches!( batcher_receiver.recv().await.unwrap(), batcher::Message::Update { .. } )); // Advance to follower view 3 (leader = participant 1). let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Send proposal + payload so verification passes. let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap(), Sha256::hash(&[b"cert_replay_payload"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Wait until the notarize for the target view is constructed. The gate is armed // in the strictly later iteration that requests certification, by which point the // notarize iteration's journal sync has completed, keeping the gate focused on // the certification and finalize appends. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Notarize(notarize)) if notarize.view() == target_view => break, _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("timed out waiting for notarize in view {target_view}"); }, } } // Send notarization to trigger certification. let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait until the first durability operation reaches the gate. Both artifacts must have // been appended by this point, but notify must still be waiting behind the sync. while pending_syncs.lock().is_empty() { reschedule().await; } let deferred = next_pending_sync(&pending_syncs); let blocked = deferred.blocked; blocked.await.expect("gated section sync was dropped"); let mut certified_before_sync = false; while let Some(msg) = resolver_receiver.recv().now_or_never().flatten() { if let MailboxMessage::Certified { notarization, success, .. } = msg && notarization.view() == target_view { assert!(success, "expected successful certification"); certified_before_sync = true; } } assert!( !certified_before_sync, "resolver observed certification before the section sync completed" ); assert!( !reporter.certifications.lock().contains_key(&target_view), "reporter observed certification before the section sync completed" ); let mut finalize_constructed = false; while let Some(msg) = batcher_receiver.recv().now_or_never().flatten() { if matches!( msg, batcher::Message::Constructed(Vote::Finalize(ref finalize)) if finalize.view() == target_view ) { finalize_constructed = true; } } assert!( !finalize_constructed, "finalize reached the batcher before the section sync completed" ); assert_eq!( pending_syncs.calls(), 1, "certification and finalize must share one section sync" ); // While the section sync is blocked, no target finalize may reach the network. let deadline = context.current() + Duration::from_millis(10); loop { select! { _ = context.sleep_until(deadline) => break, message = commonware_p2p::Receiver::recv(&mut observer_vote_receiver) => { let (_, message) = message.unwrap(); let vote: Vote = Vote::decode(message).unwrap(); assert!( !matches!( vote, Vote::Finalize(ref finalize) if finalize.view() == target_view ), "finalize reached the network before its section sync completed" ); }, } } deferred.release.send(Ok(())).unwrap(); // The batcher may report our vote, so it must only receive the // finalize after the vote is durable. let deadline = context.current() + Duration::from_secs(5); loop { select! { msg = batcher_receiver.recv() => { if matches!( msg.unwrap(), batcher::Message::Constructed(Vote::Finalize(ref finalize)) if finalize.view() == target_view ) { break; } }, _ = context.sleep_until(deadline) => { panic!("timed out waiting for durable finalize in view {target_view}"); }, } } // The resolver should observe certification only after the section sync completes. let mut certified = false; let deadline = context.current() + Duration::from_secs(5); while !certified { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { assert!(success, "expected successful certification"); certified = true; } _ => {} }, _ = context.sleep_until(deadline) => { panic!("timed out waiting for certification after section sync"); }, } } assert_eq!( reporter .certifications .lock() .get(&target_view) .map(|certification| &certification.proposal), Some(&proposal), "reporter should receive successful certification" ); // The durable finalize should be broadcast, without another section sync. let deadline = context.current() + Duration::from_secs(5); loop { select! { _ = context.sleep_until(deadline) => { panic!("timed out waiting for finalize in view {target_view}"); }, message = commonware_p2p::Receiver::recv(&mut observer_vote_receiver) => { let (_, message) = message.unwrap(); let vote: Vote = Vote::decode(message).unwrap(); if matches!( vote, Vote::Finalize(ref finalize) if finalize.view() == target_view ) { break; } }, } } assert_eq!( pending_syncs.calls(), 1, "certification and finalize used multiple section syncs" ); // Abort first voter. handle.abort(); // Second run: replay should process Artifact::Certification from journal. let certify_calls: Arc>> = Arc::new(Mutex::new(Vec::new())); let certify_tracker = certify_calls.clone(); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new( move |round, _| { certify_tracker.lock().push(round.view()); true }, )), }; let (app_actor, application) = mocks::application::Application::new(context.child("app_restarted"), app_cfg); app_actor.start(); let replay_reporter = mocks::reporter::Reporter::new( context.child("reporter_restarted"), reporter_cfg, ); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: replay_reporter.clone(), partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Wait for both replay notifications. Certification advances exactly one view. let mut replayed_certified = false; let mut advanced = false; while !(replayed_certified && advanced) { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { assert!(success, "replayed certification should be successful"); replayed_certified = true; } _ => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current >= target_view.next() { assert_eq!(current, target_view.next()); advanced = true; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("timed out waiting for restarted voter to advance past view {target_view}"); }, } } // The voter should NOT have called certify on the automaton for // target_view (it was replayed from journal). { let certified = certify_calls.lock(); assert!( !certified.contains(&target_view), "voter should not re-certify view {target_view} during replay (observed: {certified:?})" ); } let finalizes = replay_reporter.finalizes.lock(); let signers = finalizes .get(&target_view) .and_then(|payloads| payloads.get(&proposal.payload)) .expect("coalesced finalize should replay after restart"); assert!(signers.contains(&me)); assert_eq!( replay_reporter .certifications .lock() .get(&target_view) .map(|certification| &certification.proposal), Some(&proposal), "reporter should receive replayed successful certification" ); }); } #[test_traced] fn test_successful_certification_replayed_after_restart() { successful_certification_replayed_after_restart( bls12381_threshold_vrf::fixture::, ); successful_certification_replayed_after_restart( bls12381_threshold_vrf::fixture::, ); successful_certification_replayed_after_restart(bls12381_multisig::fixture::); successful_certification_replayed_after_restart(bls12381_multisig::fixture::); successful_certification_replayed_after_restart(ed25519::fixture); successful_certification_replayed_after_restart(secp256r1::fixture); } /// Tests that a failed certification (certify returns false) is correctly replayed /// from the journal after a restart. The replayed failure should trigger a timeout /// for the view (not re-certify or advance). fn failed_certification_replayed_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"failed_cert_replay".to_vec(); let partition = "failed_cert_replay".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(20)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let epoch = Epoch::new(333); // First run: certify fails (returns false). let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Custom(Box::new(|_, _| false)), }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("voter"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Advance to follower view 3. let target_view = View::new(3); let parent_payload = advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Send proposal + payload. let proposal = Proposal::new( Round::new(epoch, target_view), target_view.previous().unwrap(), Sha256::hash(&[b"failed_cert_replay_payload"]), ); let leader = participants[1].clone(); let contents = (proposal.round, parent_payload, 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); // Send notarization to trigger certification. let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait for failed certification result to be reported to resolver. loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { assert!(!success, "expected failed certification"); break; } _ => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { .. } = msg.unwrap() {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("timed out waiting for failed certification in first run"); }, } } assert!( !reporter.certifications.lock().contains_key(&target_view), "reporter should not receive failed certification" ); // Let the journal sync. context.sleep(Duration::from_millis(50)).await; while let Some(msg) = batcher_receiver.recv().now_or_never().flatten() { if let batcher::Message::Update { .. } = msg {} } // Abort first voter. handle.abort(); // Second run: replay should process Artifact::Certification(false) from journal. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app_restarted"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(5), certification_timeout: Duration::from_secs(5), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // The replayed failed certification should be reported to resolver // and the voter should NOT advance past target_view. let mut replayed_certified = false; loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Certified { notarization, success, .. } if notarization.view() == target_view => { assert!(!success, "replayed certification should be a failure"); replayed_certified = true; } _ => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() { // After replay, should be at target_view (not past it). if current == target_view && replayed_certified { break; } } }, _ = context.sleep(Duration::from_secs(5)) => { if replayed_certified { break; } panic!("timed out waiting for replayed failed certification"); }, } } assert!( replayed_certified, "resolver should receive Certified(false) during replay for view {target_view}" ); assert!( !reporter.certifications.lock().contains_key(&target_view), "reporter should not receive replayed failed certification" ); }); } #[test_traced] fn test_failed_certification_replayed_after_restart() { failed_certification_replayed_after_restart(bls12381_threshold_vrf::fixture::); failed_certification_replayed_after_restart(bls12381_threshold_vrf::fixture::); failed_certification_replayed_after_restart(bls12381_multisig::fixture::); failed_certification_replayed_after_restart(bls12381_multisig::fixture::); failed_certification_replayed_after_restart(ed25519::fixture); failed_certification_replayed_after_restart(secp256r1::fixture); } /// Tests that nullify votes and nullification certificates are correctly /// replayed from the journal after a restart. /// /// 1. First run: follower times out, votes nullify, receives nullification /// certificate. All persisted to journal. /// 2. Abort the voter. /// 3. Second run: voter replays journal and processes Artifact::Nullify and /// Artifact::Nullification entries. The resolver receives the nullification /// and the voter re-enters the same view (since it was never finalized). fn nullify_and_nullification_replayed_after_restart(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"nullify_nullification_replay".to_vec(); let partition = "nullify_nullification_replay".to_string(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let epoch = Epoch::new(333); let pending_syncs = PendingSyncs::default(); let voter_context = DelayedSyncContext { inner: context.child("voter"), pending: pending_syncs.clone(), }; // First run: trigger timeout and nullification. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter: reporter.clone(), partition: partition.clone(), epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(100), certification_timeout: Duration::from_secs(100), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(voter_context, voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); let handle = voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Advance to follower view 3. let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Gate the first-attempt nullify's journal sync. pending_syncs.arm(); mailbox.timeout(Round::new(epoch, target_view), TimeoutReason::LeaderTimeout); while pending_syncs.lock().is_empty() { reschedule().await; } let deferred = next_pending_sync(&pending_syncs); deferred .blocked .await .expect("gated nullify section sync was dropped"); while let Some(msg) = batcher_receiver.recv().now_or_never().flatten() { assert!( !matches!( msg, batcher::Message::Constructed(Vote::Nullify(ref nullify)) if nullify.view() == target_view ), "nullify reached the batcher before the section sync completed" ); } deferred.release.send(Ok(())).unwrap(); // Wait for the durable timeout-driven nullify vote. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(n)) if n.view() == target_view => { break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected nullify vote for view {target_view}"); }, } } // Send a nullification certificate for this view. let (_, nullification) = build_nullification(&schemes, Round::new(epoch, target_view), quorum); mailbox.recovered(Certificate::Nullification(nullification)); // Wait for the voter to process the nullification (advances to next view). loop { select! { msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > target_view { break; } }, _ = context.sleep(Duration::from_secs(5)) => { panic!("timed out waiting for view advance after nullification"); }, } } // Let journal sync. context.sleep(Duration::from_millis(50)).await; while let Some(msg) = batcher_receiver.recv().now_or_never().flatten() { if let batcher::Message::Update { .. } = msg {} } // Abort first voter. handle.abort(); // Second run: replay should process Artifact::Nullify and // Artifact::Nullification from journal. let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app_restarted"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition, epoch, floor: Floor::Genesis(mocks::application::genesis::(epoch)), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(1), certification_timeout: Duration::from_secs(1), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, _mailbox) = Actor::new(context.child("voter_restarted"), voter_cfg); let (resolver_sender, mut resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(8)); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(8)); let (vote_sender, _) = oracle .control(me.clone()) .register(2, TEST_QUOTA) .await .unwrap(); let (cert_sender, _) = oracle .control(me.clone()) .register(3, TEST_QUOTA) .await .unwrap(); voter.start( batcher::Mailbox::new(batcher_sender), resolver::Mailbox::new(resolver_sender), vote_sender, cert_sender, ); // Verify: resolver receives the replayed nullification. let mut replayed_nullification = false; loop { select! { msg = resolver_receiver.recv() => match msg.unwrap() { MailboxMessage::Updated { certificate: Certificate::Nullification(n), .. } if n.view() == target_view => { replayed_nullification = true; } _ => {} }, msg = batcher_receiver.recv() => { if let batcher::Message::Update { current, .. } = msg.unwrap() && current > target_view && replayed_nullification { break; } }, _ = context.sleep(Duration::from_secs(5)) => { if replayed_nullification { break; } panic!("timed out waiting for nullification replay"); }, } } assert!( replayed_nullification, "resolver should receive nullification during replay for view {target_view}" ); }); } #[test_traced] fn test_nullify_and_nullification_replayed_after_restart() { nullify_and_nullification_replayed_after_restart( bls12381_threshold_vrf::fixture::, ); nullify_and_nullification_replayed_after_restart( bls12381_threshold_vrf::fixture::, ); nullify_and_nullification_replayed_after_restart(bls12381_multisig::fixture::); nullify_and_nullification_replayed_after_restart(bls12381_multisig::fixture::); nullify_and_nullification_replayed_after_restart(ed25519::fixture); nullify_and_nullification_replayed_after_restart(secp256r1::fixture); } /// Tests that when the batcher signals a timeout reason on view update, /// the voter immediately triggers a timeout for the current view. /// /// This covers the path where `batcher.update()` returns `Some(TimeoutReason)` /// (e.g., because the leader is inactive or has already nullified the view). fn batcher_update_triggers_timeout(mut fixture: F) where S: Scheme, F: FnMut(&mut deterministic::Context, &[u8], u32) -> Fixture, { let n = 5; let quorum = quorum(n); let namespace = b"batcher_update_timeout".to_vec(); let executor = deterministic::Runner::timed(Duration::from_secs(30)); executor.start(|mut context| async move { let Fixture { participants, schemes, .. } = fixture(&mut context, &namespace, n); let oracle = start_test_network_with_peers(context.child("network"), participants.clone(), true) .await; let me = participants[0].clone(); let elector = RoundRobin::::default(); let reporter_cfg = mocks::reporter::Config { participants: participants.clone().try_into().unwrap(), scheme: schemes[0].clone(), elector: elector.clone(), }; let reporter = mocks::reporter::Reporter::new(context.child("reporter"), reporter_cfg); let relay = Arc::new(mocks::relay::Relay::::new()); let app_cfg = mocks::application::Config:: { relay: relay.clone(), me: me.clone(), propose_latency: (1.0, 0.0), verify_latency: (1.0, 0.0), certify_latency: (1.0, 0.0), should_certify: mocks::application::Certifier::Always, }; let (app_actor, application) = mocks::application::Application::new(context.child("app"), app_cfg); app_actor.start(); let voter_cfg = Config { scheme: schemes[0].clone(), elector: elector.clone().build(schemes[0].participants()), blocker: oracle.control(me.clone()), automaton: application.clone(), relay: application.clone(), reporter, partition: format!("batcher_timeout_test_{me}"), epoch: Epoch::new(333), floor: Floor::Genesis(mocks::application::genesis::(Epoch::new(333))), mailbox_size: NZUsize!(128), leader_timeout: Duration::from_secs(100), certification_timeout: Duration::from_secs(100), timeout_retry: Duration::from_mins(60), skip_budget: u64::MAX, view_retention: ViewDelta::new(10), replay_buffer: NZUsize!(1024 * 1024), write_buffer: NZUsize!(1024 * 1024), page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), }; let (voter, mut mailbox) = Actor::new(context.child("actor"), voter_cfg); let (resolver_sender, _resolver_receiver) = mailbox::new(context.child("resolver_mailbox"), NZUsize!(10)); let resolver = resolver::Mailbox::new(resolver_sender); let (batcher_sender, mut batcher_receiver) = mailbox::new(context.child("batcher_mailbox"), NZUsize!(1024)); let batcher = batcher::Mailbox::new(batcher_sender); let (vote_sender, _vote_receiver) = oracle .control(me.clone()) .register(0, TEST_QUOTA) .await .unwrap(); let (certificate_sender, _certificate_receiver) = oracle .control(me.clone()) .register(1, TEST_QUOTA) .await .unwrap(); voter.start(batcher, resolver, vote_sender, certificate_sender); // Consume initial Update. if let batcher::Message::Update { .. } = batcher_receiver.recv().await.unwrap() {} // Advance to follower view 3 using finalization. let target_view = View::new(3); advance_to_view( &mut mailbox, &mut batcher_receiver, &schemes, quorum, target_view, ) .await; // Certify view 3 to advance to view 4. let proposal = Proposal::new( Round::new(Epoch::new(333), target_view), target_view.previous().unwrap(), Sha256::hash(&[b"batcher_timeout_view3"]), ); let leader = participants[1].clone(); let contents = (proposal.round, Sha256::hash(&[b"genesis"]), 0u64).encode(); relay.broadcast(&leader, Recipients::All, (proposal.payload, contents)); mailbox.proposal(proposal.clone()); let (_, notarization) = build_notarization(&schemes, &proposal, quorum); mailbox.recovered(Certificate::Notarization(notarization)); // Wait for the Update for view 4 and simulate the batcher signaling that // the leader should be skipped. loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Update { current, .. } if current > target_view => { // Signal leader inactivity to trigger the timeout path. mailbox.timeout( Round::new(Epoch::new(333), current), TimeoutReason::Inactivity, ); break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!("expected Update for view > {target_view}"); }, } } // The voter should emit a nullify vote for view 4 quickly (not // after the 100s leader timeout) because the batcher signaled // immediate timeout. let next_view = target_view.next(); loop { select! { msg = batcher_receiver.recv() => match msg.unwrap() { batcher::Message::Constructed(Vote::Nullify(nullify)) if nullify.view() == next_view => { break; } batcher::Message::Update { .. } => {} _ => {} }, _ = context.sleep(Duration::from_secs(5)) => { panic!( "expected nullify for view {next_view} triggered by batcher timeout" ); }, } } }); } #[test_traced] fn test_batcher_update_triggers_timeout() { batcher_update_triggers_timeout(bls12381_threshold_vrf::fixture::); batcher_update_triggers_timeout(bls12381_threshold_vrf::fixture::); batcher_update_triggers_timeout(bls12381_multisig::fixture::); batcher_update_triggers_timeout(bls12381_multisig::fixture::); batcher_update_triggers_timeout(ed25519::fixture); batcher_update_triggers_timeout(secp256r1::fixture); } /// Starts a voter from `floor` over `partition`, stops it before anything /// above the floor is journaled, and restarts it from `restart`, asserting /// that it resumes at the journaled floor. Returns the restarted voter's /// handle. async fn restart_resumes_at_floor( context: &mut deterministic::Context, participants: &[PublicKey], schemes: &[S], partition: String, page_cache: CacheRef, floor: Finalization, restart: Floor, ) -> Handle<()> where S: Scheme, { let epoch = floor.epoch(); let floor_view = floor.view(); // Start from the finalized floor and stop once it has been applied let mut first = context.child("first"); let oracle = start_test_network_with_peers(first.child("network"), participants.to_vec(), true) .await; let (_mailbox, mut batcher_receiver, mut resolver_receiver, _relay, _reporter, handle) = start_voter_with_floor( &mut first, &oracle, participants, schemes, RoundRobin::::default(), VoterFloorStart { partition: partition.clone(), epoch, floor: Floor::Finalized(floor), skip_budget: u64::MAX, page_cache: page_cache.clone(), certifier: mocks::application::Certifier::Always, }, ) .await; expect_restarted_voter_at_floor( context, &mut batcher_receiver, &mut resolver_receiver, floor_view, ) .await; handle.abort(); // Restart over the same partition let mut restarted = context.child("restarted"); let oracle = start_test_network_with_peers(restarted.child("network"), participants.to_vec(), true) .await; let (_mailbox, mut batcher_receiver, mut resolver_receiver, _relay, reporter, handle) = start_voter_with_floor( &mut restarted, &oracle, participants, schemes, RoundRobin::::default(), VoterFloorStart { partition, epoch, floor: restart, skip_budget: u64::MAX, page_cache, certifier: mocks::application::Certifier::Always, }, ) .await; // The journaled floor is the active floor expect_restarted_voter_at_floor( context, &mut batcher_receiver, &mut resolver_receiver, floor_view, ) .await; assert!(reporter.finalizations.lock().contains_key(&floor_view)); handle } /// Counts the journaled finalizations at `view` in `partition`. async fn journaled_finalizations( context: deterministic::Context, partition: String, page_cache: CacheRef, scheme: &S, view: View, ) -> usize where S: Scheme, { let journal = Journal::<_, Artifact>::init( context.child("journal"), JConfig { partition, compression: None, codec_config: scheme.certificate_codec_config(), page_cache, write_buffer: NZUsize!(1024 * 1024), }, ) .await .expect("unable to initialize voter journal"); let mut replay = journal .replay(0, 0, NZUsize!(1024 * 1024), ReadOptions::DONT_CACHE) .await .expect("unable to replay voter journal"); let mut finalizations = 0; while let Some(artifact) = replay.next().await { let (_, _, _, artifact) = artifact.expect("unable to replay voter journal"); if matches!(artifact, Artifact::Finalization(_)) && artifact.view() == view { finalizations += 1; } } replay.finish().expect("unable to replay voter journal"); finalizations } #[test_traced("WARN")] fn test_voter_restart_lower_floor_keeps_journaled_floor() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_restart_lower_floor".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Start from a finalized floor over an empty journal let floor_view = View::new(5); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"journaled-floor-finalization"]), ); let (_, floor) = build_finalization(&schemes, &floor_proposal, quorum); restart_resumes_at_floor( &mut context, &participants, &schemes, "voter_restart_lower_floor".to_string(), page_cache, floor, Floor::Genesis(mocks::application::genesis::(epoch)), ) .await; }); } /// Restarting from the floor that is already journaled does not journal it /// again. #[test_traced("WARN")] fn test_voter_restart_same_floor_journals_once() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_restart_same_floor".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let partition = "voter_restart_same_floor".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Start twice from the same finalized floor let floor_view = View::new(5); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"same-floor-finalization"]), ); let (_, floor) = build_finalization(&schemes, &floor_proposal, quorum); let handle = restart_resumes_at_floor( &mut context, &participants, &schemes, partition.clone(), page_cache.clone(), floor.clone(), Floor::Finalized(floor), ) .await; handle.abort(); assert!(handle.await.is_err()); // The journal holds a single record of the floor let finalizations = journaled_finalizations( context.child("count"), partition, page_cache, &schemes[0], floor_view, ) .await; assert_eq!(finalizations, 1); }); } /// A journal holding only our own votes above the floor does not make that /// floor durable, so the floor is still journaled. #[test_traced("WARN")] fn test_voter_floor_journaled_despite_journaled_votes() { let runner = deterministic::Runner::timed(Duration::from_secs(20)); runner.start(|mut context| async move { let n = 5; let quorum = quorum(n); let epoch = Epoch::new(333); let namespace = b"voter_floor_journaled_votes".to_vec(); let Fixture { participants, schemes, .. } = ed25519::fixture(&mut context, &namespace, n); let partition = "voter_floor_journaled_votes".to_string(); let page_cache = CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE); // Persist only our own notarize above the floor let voted_view = View::new(7); let voted_proposal = Proposal::new( Round::new(epoch, voted_view), voted_view.previous().unwrap(), Sha256::hash(&[b"vote-above-floor"]), ); let notarize = Notarize::sign(&schemes[0], voted_proposal).unwrap(); seed_voter_journal_artifacts( context.child("seed_vote"), partition.clone(), page_cache.clone(), &schemes[0], voted_view, vec![Artifact::Notarize(notarize)], ) .await; // Start from a finalized floor below that vote let floor_view = View::new(5); let floor_proposal = Proposal::new( Round::new(epoch, floor_view), floor_view.previous().unwrap(), Sha256::hash(&[b"floor-below-vote"]), ); let (_, floor) = build_finalization(&schemes, &floor_proposal, quorum); restart_resumes_at_floor( &mut context, &participants, &schemes, partition, page_cache, floor, Floor::Genesis(mocks::application::genesis::(epoch)), ) .await; }); } }