use crate::{ telemetry::metrics::raw::Gauge, utils::{extract_panic_message, supervision::Tree}, Error, }; use commonware_utils::{ channel::oneshot, sync::{Mutex, Once}, }; use futures::{ future::{select, Either}, pin_mut, stream::{AbortHandle, Abortable, Aborted}, FutureExt as _, }; use std::{ any::Any, future::Future, panic::{resume_unwind, AssertUnwindSafe}, pin::Pin, sync::Arc, task::{Context, Poll}, }; use tracing::error; /// Handle to an asynchronous result. /// /// Handles returned by [`crate::Spawner::spawn`] abort the spawned task. Completion handles only /// stop waiting when aborted, resolving to [`Error::Aborted`]; they do not cancel the underlying /// work. pub struct Handle where T: Send + 'static, { state: HandleState, } /// Distinguishes handles that own spawned work from handles that only wait on completion. enum HandleState where T: Send + 'static, { Task { receiver: oneshot::Receiver>, abort_handle: AbortHandle, metric: MetricHandle, }, Completion { future: Abortable>, abort_handle: AbortHandle, }, } /// Normalizes receiver-backed and future-backed completions behind one abortable future. enum Completion where T: Send + 'static, { Receiver(oneshot::Receiver>), Future(Pin> + Send + 'static>>), } impl Unpin for Completion where T: Send + 'static {} impl Future for Completion where T: Send + 'static, { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { match &mut *self { Self::Receiver(receiver) => Pin::new(receiver) .poll(cx) .map(|result| result.unwrap_or(Err(Error::Closed))), Self::Future(future) => future.as_mut().poll(cx), } } } impl Handle where T: Send + 'static, { #[inline(always)] pub(crate) fn init( f: F, metric: MetricHandle, panicker: Panicker, tree: Arc, ) -> (impl Future, Self) where F: Future + Send + 'static, { // Initialize channels to handle result/abort let (sender, receiver) = oneshot::channel(); let (abort_handle, abort_registration) = AbortHandle::new_pair(); // Wrap the future with panic catching, abort support, and cleanup. // // Everything is done in a single async block (and the function is marked // #[inline(always)]) so that stack usage is `size_of(F) + constant` rather than // `N * size_of(F)` (which is what a combinator chain produces in debug builds). let metric_handle = metric.clone(); let task = async move { // Run future with panic catching and abort support let result = Abortable::new(AssertUnwindSafe(f).catch_unwind(), abort_registration).await; // Handle result match result { Ok(Ok(result)) => { let _ = sender.send(Ok(result)); } Ok(Err(panic)) => { panicker.notify(panic); let _ = sender.send(Err(Error::Exited)); } Err(Aborted) => {} } // Mark the task as aborted and abort all descendants. tree.abort(); // Finish the metric. metric_handle.finish(); }; ( task, Self { state: HandleState::Task { receiver, abort_handle, metric, }, }, ) } /// Returns a handle backed by a completion receiver. pub fn from_receiver(receiver: oneshot::Receiver>) -> Self { let (abort_handle, abort_registration) = AbortHandle::new_pair(); Self { state: HandleState::Completion { future: Abortable::new(Completion::Receiver(receiver), abort_registration), abort_handle, }, } } /// Returns a handle backed by a completion future. pub fn from_future(future: F) -> Self where F: Future> + Send + 'static, { let (abort_handle, abort_registration) = AbortHandle::new_pair(); Self { state: HandleState::Completion { future: Abortable::new(Completion::Future(Box::pin(future)), abort_registration), abort_handle, }, } } /// Returns a handle that is already complete. pub fn ready(result: Result) -> Self { let (sender, receiver) = oneshot::channel(); let _ = sender.send(result); Self::from_receiver(receiver) } /// Returns a handle that resolves to [`Error::Closed`] without spawning work. pub(crate) fn closed(metric: MetricHandle) -> Self { // Mark the task as finished immediately so gauges remain accurate. metric.finish(); // Create a receiver that will yield `Err(Error::Closed)` when awaited. let (sender, receiver) = oneshot::channel(); drop(sender); Self::from_receiver(receiver) } /// Abort the spawned task or stop waiting for a completion. pub fn abort(&self) { match &self.state { HandleState::Task { abort_handle, metric, .. } => { abort_handle.abort(); // We might never poll the future again after aborting it, so run the // metric cleanup right away. metric.finish(); } HandleState::Completion { abort_handle, .. } => { abort_handle.abort(); } } } /// Returns a helper that aborts the task and updates metrics consistently. pub(crate) fn aborter(&self) -> Option { match &self.state { HandleState::Task { abort_handle, metric, .. } => Some(Aborter::new(abort_handle.clone(), metric.clone())), HandleState::Completion { .. } => None, } } } impl Future for Handle where T: Send + 'static, { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { match &mut self.state { HandleState::Task { receiver, .. } => Pin::new(receiver) .poll(cx) .map(|result| result.unwrap_or_else(|_| Err(Error::Closed))), HandleState::Completion { future, .. } => Pin::new(future) .poll(cx) .map(|result| result.unwrap_or(Err(Error::Aborted))), } } } /// Tracks the metric state associated with a spawned task handle. #[derive(Clone)] pub(crate) struct MetricHandle { gauge: Gauge, finished: Arc, } impl MetricHandle { /// Increments the supplied gauge and returns a handle responsible for /// eventually decrementing it. pub(crate) fn new(gauge: Gauge) -> Self { gauge.inc(); Self { gauge, finished: Arc::new(Once::new()), } } /// Marks the task handle as completed and decrements the gauge once. /// /// This method is idempotent, additional calls are ignored so completion /// and abort paths can invoke it independently. pub(crate) fn finish(&self) { let gauge = self.gauge.clone(); self.finished.call_once(move || { gauge.dec(); }); } } /// A panic emitted by a spawned task. pub type Panic = Box; /// Notifies the runtime when a spawned task panics, so it can propagate the failure. #[derive(Clone)] pub(crate) struct Panicker { catch: bool, sender: Arc>>>, } impl Panicker { /// Creates a new [Panicker]. pub(crate) fn new(catch: bool) -> (Self, Panicked) { let (sender, receiver) = oneshot::channel(); let panicker = Self { catch, sender: Arc::new(Mutex::new(Some(sender))), }; let panicked = Panicked { receiver }; (panicker, panicked) } /// Returns whether the [Panicker] is configured to catch panics. #[commonware_macros::stability(ALPHA)] pub(crate) const fn catch(&self) -> bool { self.catch } /// Notifies the [Panicker] that a panic has occurred. pub(crate) fn notify(&self, panic: Box) { // Log the panic let err = extract_panic_message(&*panic); error!(?err, "task panicked"); // If we are catching panics, just return if self.catch { return; } // If we've already sent a panic, ignore the new one let mut sender = self.sender.lock(); let Some(sender) = sender.take() else { return; }; // Send the panic let _ = sender.send(panic); } } /// A handle that will be notified when a panic occurs. pub(crate) struct Panicked { receiver: oneshot::Receiver, } impl Panicked { /// Polls a task that should be interrupted by a panic. pub(crate) async fn interrupt(self, task: Fut) -> Fut::Output where Fut: Future, { // Wait for task to complete or panic let panicked = self.receiver; pin_mut!(panicked); pin_mut!(task); match select(panicked, task).await { Either::Left((panic, task)) => match panic { // If there is a panic, resume the unwind Ok(panic) => { resume_unwind(panic); } // If there can never be a panic (oneshot is closed), wait for the task to complete // and return the output Err(_) => task.await, }, Either::Right((output, _)) => { // Return the output output } } } } /// Couples an [`AbortHandle`] with its metric handle so aborted tasks clean up gauges. pub(crate) struct Aborter { inner: AbortHandle, metric: MetricHandle, } impl Aborter { /// Creates a new [`Aborter`] for the provided abort handle and metric handle. pub(crate) const fn new(inner: AbortHandle, metric: MetricHandle) -> Self { Self { inner, metric } } /// Aborts the task and records completion in the metric gauge. pub(crate) fn abort(self) { self.inner.abort(); // We might never poll the future again after aborting it, so run the // metric cleanup right away self.metric.finish(); } } #[cfg(test)] mod tests { use super::Handle; use crate::{deterministic, Error, Metrics as _, Runner, Spawner, Supervisor as _}; use commonware_utils::channel::oneshot; use futures::future; const METRIC_PREFIX: &str = "runtime_tasks_running{"; fn running_tasks_for_label(metrics: &str, label: &str) -> Option { let label_fragment = format!("name=\"{label}\""); metrics.lines().find_map(|line| { if line.starts_with(METRIC_PREFIX) && line.contains(&label_fragment) { line.rsplit_once(' ') .and_then(|(_, value)| value.trim().parse::().ok()) } else { None } }) } #[test] fn tasks_running_decreased_after_completion() { const LABEL: &str = "tasks_running_after_completion"; let runner = deterministic::Runner::default(); runner.start(|context| async move { let handle = context.child(LABEL).spawn(|_| async move { "done" }); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "expected tasks_running gauge to be 1 before completion: {metrics}", ); let output = handle.await.expect("task failed"); assert_eq!(output, "done"); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(0), "expected tasks_running gauge to return to 0 after completion: {metrics}", ); }); } #[test] fn tasks_running_unchanged_when_handle_dropped() { const LABEL: &str = "tasks_running_unchanged"; let runner = deterministic::Runner::default(); runner.start(|context| async move { let handle = context.child(LABEL).spawn(|_| async move { future::pending::<()>().await; }); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "expected tasks_running gauge to be 1 before dropping handle: {metrics}", ); drop(handle); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "dropping handle should not finish metrics: {metrics}", ); }); } #[test] fn tasks_running_decreased_immediately_on_abort_via_handle() { const LABEL: &str = "tasks_running_abort_via_handle"; let runner = deterministic::Runner::default(); runner.start(|context| async move { let handle = context.child(LABEL).spawn(|_| async move { future::pending::<()>().await; }); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "expected tasks_running gauge to be 1 before abort: {metrics}", ); handle.abort(); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(0), "expected tasks_running gauge to return to 0 after abort: {metrics}", ); }); } #[test] fn completion_handle_abort_stops_waiting() { deterministic::Runner::default().start(|_| async move { let (sender, receiver) = oneshot::channel(); let handle = Handle::from_receiver(receiver); handle.abort(); assert!(sender.send(Ok(())).is_ok()); assert!(matches!(handle.await, Err(Error::Aborted))); }); } #[test] fn tasks_running_decreased_after_blocking_completion() { const LABEL: &str = "tasks_running_after_blocking_completion"; let runner = deterministic::Runner::default(); runner.start(|context| async move { let blocking_handle = context.child(LABEL).shared(true).spawn(|_| async move { // Simulate some blocking work 42 }); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "expected tasks_running gauge to be 1 while blocking task runs: {metrics}", ); let result = blocking_handle.await.expect("blocking task failed"); assert_eq!(result, 42); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(0), "expected tasks_running gauge to return to 0 after blocking task completes: {metrics}", ); }); } #[test] fn tasks_running_decreased_immediately_on_abort_via_aborter() { const LABEL: &str = "tasks_running_abort_via_aborter"; let runner = deterministic::Runner::default(); runner.start(|context| async move { let handle = context.child(LABEL).spawn(|_| async move { future::pending::<()>().await; }); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(1), "expected tasks_running gauge to be 1 before abort: {metrics}", ); let aborter = handle.aborter().unwrap(); aborter.abort(); let metrics = context.encode(); assert_eq!( running_tasks_for_label(&metrics, LABEL), Some(0), "expected tasks_running gauge to return to 0 after abort: {metrics}", ); }); } }