//! Track pending consumer deliveries for resolver implementations. //! //! Resolvers often need the same delivery lifecycle: keep a fetch alive while //! `Consumer::deliver` validates a response, abort that validation if the fetch //! is pruned, and reuse a cached response for subscribers that were added //! while validation was in progress. This module owns that lifecycle without //! making assumptions about how data is fetched. use crate::{Consumer, Delivery, Outcome}; use ahash::AHashMap; use commonware_macros::select; use commonware_utils::{channel::oneshot, futures::Pool}; use futures::future::Aborted; use std::collections::hash_map::Entry as HashMapEntry; use tracing::debug; /// Completed consumer validation for a delivery. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Completion { /// Resolver-specific context associated with the delivery attempt. pub context: Context, /// Key and subscribers that were passed to the consumer. pub delivery: Delivery, /// Consumer disposition for the delivered response, or `None` if the /// consumer dropped the sender without reporting one. pub outcome: Option, /// Whether the tracked fetch was removed before this completion was polled. /// Its verdict applies only to the source, not to any current fetch for the key. pub canceled: bool, /// Whether the cached response was already accepted when this delivery began. pub previously_accepted: bool, } // Delivery futures handle cancellation so verdicts sent before receiver closure can still // reach source accounting. Canceled deliveries without a verdict return `Aborted`. type DeliveryPool = Pool<'static, Result, Aborted>>; // Cached response that can be redelivered while it is accepted or still unjudged. struct Response { context: Context, value: V, accepted: bool, } // Per-key delivery state retained while a resolver fetch is active. struct Entry { delivery: Option>, response: Option>, state: Option, } impl Entry { const fn new(state: State) -> Self { Self { delivery: None, response: None, state: Some(state), } } } /// Tracks in-flight consumer deliveries keyed by resolver key. /// /// `Context` carries resolver-specific metadata back to the caller when /// validation completes. /// /// `State` holds per-key resolver state that is dropped when the key is pruned. /// While the key stays tracked, the resolver can read, update, or take it. pub struct Tracker where Con: Consumer, Con::Value: Clone + Send + 'static, Context: Clone + Send + 'static, { entries: AHashMap>, deliveries: DeliveryPool, consumer: Con, } impl Tracker where Con: Consumer, Con::Value: Clone + Send + 'static, Context: Clone + Send + 'static, { /// Create an empty tracker backed by the provided consumer. pub fn new(consumer: Con) -> Self { Self { entries: AHashMap::new(), deliveries: Pool::default(), consumer, } } /// Returns true when the key has an active tracked fetch. pub fn contains(&self, key: &Con::Key) -> bool { self.entries.contains_key(key) } /// Start tracking a key. /// /// Returns `true` when the key was inserted. If the key is already tracked, /// this leaves the existing entry untouched and returns `false`. pub(crate) fn insert_with_state(&mut self, key: Con::Key, state: State) -> bool { match self.entries.entry(key) { HashMapEntry::Vacant(entry) => { entry.insert(Entry::new(state)); true } HashMapEntry::Occupied(_) => false, } } /// Remove a key and abort any in-progress delivery for it. /// /// Returns true if the key was present. A verdict accepted before cancellation /// closes its receiver is returned as a canceled [`Completion`]. pub fn remove(&mut self, key: &Con::Key) -> bool { self.entries.remove(key).is_some() } /// Remove a key, aborting any in-progress delivery, and return its state. /// /// Returns `None` if the key was absent. The inner `Option` is `None` if the /// state was already taken while the key stayed active. pub(crate) fn remove_with_state(&mut self, key: &Con::Key) -> Option> { self.entries.remove(key).map(|entry| entry.state) } /// Take the key's state without removing the tracked key. /// /// Returns `None` when the key is absent or the state was already taken. pub(crate) fn take_state(&mut self, key: &Con::Key) -> Option { self.entries .get_mut(key) .and_then(|entry| entry.state.take()) } /// Mutably borrow the key's state without taking it. /// /// Returns `None` when the key is absent or the state was already taken. pub(crate) fn state_mut(&mut self, key: &Con::Key) -> Option<&mut State> { self.entries .get_mut(key) .and_then(|entry| entry.state.as_mut()) } /// Retain only entries for which the predicate returns true. /// /// Dropped entries abort in-progress deliveries. Returns the number of /// removed entries. pub fn retain bool>(&mut self, mut predicate: F) -> usize { self.entries.extract_if(|key, _| !predicate(key)).count() } /// Remove all entries and abort all in-progress deliveries. /// /// Returns the number of entries removed. pub fn drain(&mut self) -> usize { let count = self.entries.len(); self.entries.clear(); count } /// Deliver a newly received response to the consumer. /// /// The response is cached so that later retained subscribers can be /// redelivered the same bytes with [`redeliver`](Self::redeliver) once the /// consumer accepts it or drops its verdict. Panics if the key is not tracked. pub fn deliver( &mut self, delivery: Delivery, context: Context, value: Con::Value, ) { let entry = self.entries.get_mut(&delivery.key).expect("delivery entry"); entry.response = Some(Response { context: context.clone(), value: value.clone(), accepted: false, }); self.push_delivery(delivery, context, value); } /// Deliver the cached response to another set of subscribers. /// /// This is intended for subscribers added while an earlier validation was /// still pending. The cached response is either accepted or still unjudged /// because the consumer dropped the earlier verdict. Panics if the key is not /// tracked or no response is cached. pub fn redeliver(&mut self, delivery: Delivery) { let (context, value) = { let entry = self.entries.get(&delivery.key).expect("delivery entry"); let response = entry.response.as_ref().expect("response"); (response.context.clone(), response.value.clone()) }; self.push_delivery(delivery, context, value); } /// Returns true if the cached response for this key has been accepted. pub fn response_accepted(&self, key: &Con::Key) -> bool { self.entries .get(key) .and_then(|entry| entry.response.as_ref()) .is_some_and(|response| response.accepted) } /// Mark the cached response accepted by the consumer. /// /// Panics if the key is not tracked or no response is cached. pub fn accept_response(&mut self, key: &Con::Key) { let entry = self.entries.get_mut(key).expect("delivery entry"); let response = entry.response.as_mut().expect("response"); response.accepted = true; } /// Drop the cached response without removing the tracked key. /// /// Use this when a response is invalid or does not satisfy every delivered /// subscriber and the resolver wants to retry the key. pub fn discard_response(&mut self, key: &Con::Key) { if let Some(entry) = self.entries.get_mut(key) { entry.response = None; } } /// Wait for the next consumer validation result. /// /// Returns [`Aborted`] when cancellation closes the receiver without an accepted /// verdict. Otherwise, canceled deliveries return their verdict for source /// accounting without affecting any current fetch for the key. Active deliveries /// clear their slot so the key can be retried or redelivered. A verdict sender /// dropped by the consumer completes with no outcome. pub async fn next_completion( &mut self, ) -> Result, Aborted> { let completed = self.deliveries.next_completed().await?; // Cancellation belongs to the removed entry. Only an active completion can // clear a delivery slot; a canceled key may already have a replacement entry. if !completed.canceled { let entry = self .entries .get_mut(&completed.delivery.key) .expect("delivery entry"); entry.delivery = None; } Ok(completed) } // Start a consumer validation attempt whose cancellation is owned by its entry. fn push_delivery( &mut self, delivery: Delivery, context: Context, value: Con::Value, ) { let entry = self.entries.get_mut(&delivery.key).expect("delivery entry"); let previously_accepted = entry.response.as_ref().expect("response").accepted; let completed = delivery.clone(); let mut consumer = self.consumer.clone(); let mut receiver = consumer.deliver(delivery, value); let (cancel, canceled) = oneshot::channel::<()>(); self.deliveries.push(async move { // Cancellation takes precedence over a ready verdict so feedback from a // removed entry can never complete a replacement. Closing before reading // preserves successful sends without waiting for an unfinished verdict. let (outcome, canceled) = select! { _ = canceled => { receiver.close(); (receiver.try_recv().ok().map(Into::into), true) }, outcome = &mut receiver => { let outcome = match outcome { Ok(outcome) => Some(outcome.into()), Err(_) => { debug!(key = ?completed.key, "consumer dropped delivery without a verdict"); None } }; (outcome, false) }, }; if canceled && outcome.is_none() { return Err(Aborted); } Ok(Completion { context, delivery: completed, outcome, canceled, previously_accepted, }) }); assert!(entry.delivery.replace(cancel).is_none()); } } impl Tracker where Con: Consumer, Con::Value: Clone + Send + 'static, Context: Clone + Send + 'static, { /// Start tracking a key without any resolver-specific state. /// /// Returns `true` when the key was inserted. If the key is already tracked, /// this leaves the existing entry untouched and returns `false`. pub fn insert(&mut self, key: Con::Key) -> bool { self.insert_with_state(key, ()) } } #[cfg(test)] mod tests { use super::*; use crate::{ Resolution, p2p::mocks::{Consumer as MockConsumer, Key as MockKey}, }; use bytes::Bytes; use commonware_runtime::{Runner as _, deterministic::Runner}; use commonware_utils::{ channel::{fallible::FallibleExt, mpsc, oneshot}, non_empty_vec, }; use futures::FutureExt as _; type TestTracker = Tracker, u8>; type StateTracker = Tracker, u8, u8>; fn delivery(key: MockKey) -> Delivery { Delivery { key, subscribers: non_empty_vec![((), tracing::Span::none())], } } #[derive(Clone)] struct PendingConsumer { sender: mpsc::UnboundedSender>, } impl PendingConsumer { fn new() -> (Self, mpsc::UnboundedReceiver>) { let (sender, receiver) = mpsc::unbounded_channel(); (Self { sender }, receiver) } } impl Consumer for PendingConsumer { type Key = MockKey; type Value = Bytes; type Subscriber = (); type Outcome = bool; fn deliver( &mut self, _delivery: Delivery, _value: Self::Value, ) -> oneshot::Receiver { let (sender, receiver) = oneshot::channel(); self.sender.send_lossy(sender); receiver } } #[test] fn test_insert_contains_remove_round_trip() { let runner = Runner::default(); runner.start(|_| async move { let mut tracker = TestTracker::new(MockConsumer::dummy()); assert!(!tracker.contains(&MockKey(1))); assert!(tracker.insert(MockKey(1))); assert!(tracker.contains(&MockKey(1))); assert!(!tracker.insert(MockKey(1))); assert!(tracker.remove(&MockKey(1))); assert!(!tracker.contains(&MockKey(1))); assert!(!tracker.remove(&MockKey(1))); }); } #[test] fn test_state_round_trip() { let runner = Runner::default(); runner.start(|_| async move { // An untracked key has no state. let mut tracker = StateTracker::new(MockConsumer::dummy()); assert_eq!(tracker.state_mut(&MockKey(1)).map(|state| *state), None); // Inserted state can be read and updated in place. assert!(tracker.insert_with_state(MockKey(1), 5)); assert_eq!(tracker.state_mut(&MockKey(1)).map(|state| *state), Some(5)); *tracker .state_mut(&MockKey(1)) .expect("inserted key must have state") = 7; assert_eq!(tracker.state_mut(&MockKey(1)).map(|state| *state), Some(7)); // Taking the state leaves the key tracked without state, so removing // the key returns no state. assert_eq!(tracker.take_state(&MockKey(1)), Some(7)); assert!(tracker.contains(&MockKey(1))); assert_eq!(tracker.state_mut(&MockKey(1)), None); assert_eq!(tracker.remove_with_state(&MockKey(1)), Some(None)); assert_eq!(tracker.state_mut(&MockKey(1)).map(|state| *state), None); }); } #[test] fn test_deliver_completes_with_context_and_consumer_result() { let runner = Runner::default(); runner.start(|_| async move { let (consumer, mut events) = MockConsumer::::new(); let mut tracker = TestTracker::new(consumer); let key = MockKey(7); let value = Bytes::from("data"); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 9, value.clone()); let completed = tracker .next_completion() .await .expect("delivery should complete"); assert_eq!(completed.context, 9); assert_eq!(completed.delivery.key, key); assert!(!completed.canceled); assert!(!completed.previously_accepted); assert_eq!( completed.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); let (delivered_key, delivered_value) = events.recv().await.unwrap(); assert_eq!(delivered_key, key); assert_eq!(delivered_value, value); }); } #[test] fn test_remove_aborts_in_flight_delivery() { let runner = Runner::default(); runner.start(|_| async move { let (consumer, mut senders) = PendingConsumer::new(); let mut tracker = Tracker::<_, u8>::new(consumer); let key = MockKey(1); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 2, Bytes::from("v")); let sender = senders.recv().await.unwrap(); assert!(tracker.remove(&key)); assert!(matches!(tracker.next_completion().await, Err(Aborted))); assert!(sender.is_closed()); }); } #[test] fn test_canceled_delivery_cannot_complete_reinserted_key() { #[derive(Clone, Copy)] enum VerdictTiming { BeforeRemoval, AfterRemoval, AfterClose, } // Every removal path must preserve accepted feedback and isolate the replacement. for removal in 0..4 { for timing in [ VerdictTiming::BeforeRemoval, VerdictTiming::AfterRemoval, VerdictTiming::AfterClose, ] { for polled in [false, true] { Runner::default().start(move |_| async move { // Keep the old verdict pending or make it ready without consuming it. let (consumer, mut senders) = PendingConsumer::new(); let mut tracker = Tracker::::new(consumer); let key = MockKey(1); assert!(tracker.insert_with_state(key.clone(), 10)); tracker.deliver(delivery(key.clone()), 1, Bytes::from("old")); let mut old_sender = Some(senders.recv().await.unwrap()); if polled { assert!(tracker.next_completion().now_or_never().is_none()); } if matches!(timing, VerdictTiming::BeforeRemoval) { old_sender.take().unwrap().send(true).unwrap(); } // Reuse the key before polling the canceled delivery. match removal { 0 => assert!(tracker.remove(&key)), 1 => assert_eq!(tracker.remove_with_state(&key), Some(Some(10))), 2 => assert_eq!(tracker.retain(|_| false), 1), _ => assert_eq!(tracker.drain(), 1), } if matches!(timing, VerdictTiming::AfterRemoval) { old_sender.take().unwrap().send(true).unwrap(); } assert!(tracker.insert_with_state(key.clone(), 20)); tracker.deliver(delivery(key.clone()), 2, Bytes::from("new")); let new_sender = senders.recv().await.unwrap(); // Cancellation closes pending verdicts and preserves accepted // ones without touching the replacement's state or delivery. let result = tracker.next_completion().await; if matches!(timing, VerdictTiming::AfterClose) { assert!(matches!(result, Err(Aborted))); assert!(old_sender.take().unwrap().send(true).is_err()); } else { let completed = result.unwrap(); assert!(completed.canceled); assert!(!completed.previously_accepted); assert_eq!(completed.context, 1); assert_eq!( completed.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); } assert_eq!(tracker.state_mut(&key).map(|state| *state), Some(20)); assert!(tracker.entries[&key].delivery.is_some()); assert!(tracker.next_completion().now_or_never().is_none()); // Only the replacement's verdict may complete the new entry. new_sender.send(true).unwrap(); let completed = tracker.next_completion().await.unwrap(); assert!(!completed.canceled); assert_eq!(completed.context, 2); assert_eq!(completed.delivery.key, key); assert_eq!( completed.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); assert!(tracker.deliveries.is_empty()); }); } } } } #[test] fn test_dropped_verdict_completes_without_outcome_and_redelivers() { let runner = Runner::default(); runner.start(|_| async move { let (consumer, mut senders) = PendingConsumer::new(); let mut tracker = Tracker::::new(consumer); let key = MockKey(3); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 4, Bytes::from("unjudged")); drop(senders.recv().await.unwrap()); let completed = tracker .next_completion() .await .expect("dropped verdict should complete"); assert_eq!(completed.context, 4); assert_eq!(completed.delivery.key, key); assert_eq!(completed.outcome, None); assert!(!tracker.response_accepted(&key)); // The unjudged response can still be handed to other subscribers. tracker.redeliver(delivery(key.clone())); senders.recv().await.unwrap().send(true).unwrap(); let judged = tracker .next_completion() .await .expect("redelivery should complete"); assert_eq!(judged.context, 4); assert_eq!( judged.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); }); } #[test] fn test_redeliver_reuses_accepted_response_for_new_subscribers() { let runner = Runner::default(); runner.start(|_| async move { let (consumer, mut events) = MockConsumer::::new(); let mut tracker = TestTracker::new(consumer); let key = MockKey(5); let value = Bytes::from("first"); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 3, value.clone()); let completed = tracker .next_completion() .await .expect("first delivery should complete"); assert!(!completed.previously_accepted); assert_eq!( completed.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); tracker.accept_response(&key); assert!(tracker.response_accepted(&key)); tracker.redeliver(delivery(key.clone())); let redelivered = tracker .next_completion() .await .expect("redelivery should complete"); assert_eq!(redelivered.context, 3); assert_eq!(redelivered.delivery.key, key); assert!(redelivered.previously_accepted); assert_eq!( redelivered.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); let first = events.recv().await.unwrap(); let second = events.recv().await.unwrap(); assert_eq!(first, (key.clone(), value.clone())); assert_eq!(second, (key, value)); }); } #[test] #[should_panic(expected = "response")] fn test_redeliver_requires_cached_response() { let runner = Runner::default(); runner.start(|_| async move { let (consumer, _events) = MockConsumer::::new(); let mut tracker = TestTracker::new(consumer); let key = MockKey(7); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 3, Bytes::from("first")); let completed = tracker .next_completion() .await .expect("first delivery should complete"); assert_eq!( completed.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); tracker.discard_response(&key); tracker.redeliver(delivery(key)); }); } #[test] fn test_rejected_response_can_be_discarded_and_replaced() { let runner = Runner::default(); runner.start(|_| async move { let (mut consumer, _events) = MockConsumer::::new(); let key = MockKey(8); consumer.add_expected(key.clone(), Bytes::from("good")); let mut tracker = TestTracker::new(consumer); tracker.insert(key.clone()); tracker.deliver(delivery(key.clone()), 1, Bytes::from("bad")); let rejected = tracker .next_completion() .await .expect("rejected delivery should complete"); assert_eq!(rejected.outcome, Some(Outcome::Invalid)); tracker.discard_response(&key); assert!(!tracker.response_accepted(&key)); tracker.deliver(delivery(key.clone()), 2, Bytes::from("good")); let accepted = tracker .next_completion() .await .expect("accepted delivery should complete"); assert_eq!(accepted.context, 2); assert_eq!( accepted.outcome, Some(Outcome::Resolved(Resolution::Complete)) ); }); } }