Skip to main content

gwr_engine/
executor.rs

1// Copyright (c) 2023 Graphcore Ltd. All rights reserved.
2
3//! Single-threaded task polling and waker ownership.
4//!
5//! Spawning puts an `Rc<Task>` in `new_tasks`. Each step moves queued handles
6//! into a reusable buffer and drains it, polling each handle's future if it
7//! has not already completed. Several queued handles can refer to the same
8//! task, so a pending future may be polled more than once in a step.
9//!
10//! Each drained handle is dropped after use. A pending task stays alive only
11//! while another strong reference exists, such as a queued handle or a saved
12//! waker. There is no separate collection retaining all pending tasks.
13//!
14//! The borrowed/owned distinction applies to wakers; both refer to the same
15//! heap-allocated task. There are two raw-waker representations:
16//!
17//! - The polling context uses `BORROWED_VTABLE`. Its data points to the local
18//!   `Rc<Task>` handle, not directly to the task. `waker_for_task` returns a
19//!   `WakerRef` whose lifetime is tied to that handle's borrow, preventing safe
20//!   code from retaining this representation after the handle goes away.
21//!   `ManuallyDrop` suppresses destruction of the temporary waker. Constructing
22//!   this context therefore needs no extra reference-count increment/decrement.
23//! - Cloning the context waker clones the borrowed `Rc` and passes that new
24//!   owner through `Rc::into_raw`. The result uses `VTABLE`, points directly to
25//!   the task, and owns one strong reference. It can outlive the poll. Further
26//!   clones each acquire another reference; dropping an owned waker releases
27//!   its reference. Consuming it with `wake()` queues a task reference and
28//!   releases the reference owned by that waker.
29//!
30//! The lifetimes enforce the borrowed context's scope, but choosing the correct
31//! data pointer and vtable is an unsafe-code invariant: only owned-waker
32//! callbacks may reconstruct an `Rc<Task>` with `Rc::from_raw`, and their input
33//! must originate from `Rc::into_raw`. A borrowed pointer cannot be treated as
34//! an owned reference merely by suppressing its destructor. Both vtables
35//! support `wake_by_ref` by cloning one task reference into `new_tasks` while
36//! preserving the source waker. The borrowed vtable rejects consuming wake and
37//! drop, which safe use of `WakerRef` prevents. Those callbacks panic if
38//! called.
39//!
40//! Completion removes the future from the task, so a later queued wake cannot
41//! poll it again. The task is destroyed when its last strong reference is
42//! released. These task references are separate from `Rc<ExecutorState>`, which
43//! keeps the shared executor state alive.
44
45use std::cell::{Cell, RefCell};
46use std::future::Future;
47use std::pin::Pin;
48use std::rc::Rc;
49use std::sync::Arc;
50use std::sync::atomic::{AtomicBool, Ordering};
51use std::sync::mpsc::{self, RecvTimeoutError};
52use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
53use std::time::Duration;
54use std::{mem, thread};
55
56use futures::task::WakerRef;
57use gwr_track::entity::Entity;
58use rand::SeedableRng;
59use rand::rngs::StdRng;
60use rand::seq::SliceRandom;
61
62use crate::time::clock::Clock;
63use crate::time::simtime::SimTime;
64use crate::types::SimResult;
65
66fn no_op(_: *const ()) {
67    unreachable!("borrowed executor waker must not be consumed or dropped");
68}
69
70unsafe fn drop_task(data: *const ()) {
71    // SAFETY: VTABLE supplies an owned task reference created by Rc::into_raw.
72    // Dropping the reconstructed Rc releases that reference.
73    unsafe {
74        drop(Rc::from_raw(data as *const Task));
75    }
76}
77
78static VTABLE: RawWakerVTable =
79    RawWakerVTable::new(clone_raw_waker, wake_task, wake_task_by_ref, drop_task);
80static BORROWED_VTABLE: RawWakerVTable = RawWakerVTable::new(
81    clone_borrowed_waker,
82    no_op,
83    wake_borrowed_task_by_ref,
84    no_op,
85);
86
87fn waker_for_task(task: &Rc<Task>) -> WakerRef<'_> {
88    // SAFETY: the data points to the Rc handle, which remains borrowed for the
89    // lifetime of the returned WakerRef. Cloning produces an owned waker with
90    // its own Rc reference and vtable. Wake-by-reference clones into the queue.
91    let raw = RawWaker::new(std::ptr::from_ref(task).cast(), &BORROWED_VTABLE);
92    let waker = mem::ManuallyDrop::new(unsafe { Waker::from_raw(raw) });
93    WakerRef::new_unowned(waker)
94}
95
96unsafe fn clone_borrowed_waker(data: *const ()) -> RawWaker {
97    // SAFETY: only BORROWED_VTABLE calls this function, and its data points to
98    // the live Rc handle borrowed by waker_for_task.
99    let task = unsafe { &*data.cast::<Rc<Task>>() };
100    RawWaker::new(Rc::into_raw(task.clone()).cast(), &VTABLE)
101}
102
103unsafe fn wake_borrowed_task_by_ref(data: *const ()) {
104    // SAFETY: BORROWED_VTABLE points to the live Rc handle borrowed by
105    // waker_for_task. The queue gets a new reference, leaving that handle
106    // intact.
107    let task = unsafe { &*data.cast::<Rc<Task>>() };
108    task.executor_state
109        .new_tasks
110        .borrow_mut()
111        .push(task.clone());
112}
113
114unsafe fn wake_task_by_ref(data: *const ()) {
115    // SAFETY: VTABLE points to an owned reference from Rc::into_raw.
116    // ManuallyDrop preserves the source waker's ownership of that reference.
117    let task = mem::ManuallyDrop::new(unsafe { Rc::from_raw(data.cast::<Task>()) });
118    task.executor_state
119        .new_tasks
120        .borrow_mut()
121        .push(Rc::clone(&task));
122}
123
124unsafe fn clone_raw_waker(data: *const ()) -> RawWaker {
125    // SAFETY: VTABLE is used only for owned pointers from Rc::into_raw.
126    // Keep the original reference owned by the source waker.
127    unsafe {
128        let rc_task = mem::ManuallyDrop::new(Rc::from_raw(data as *const Task));
129        let clone = Rc::clone(&rc_task);
130        let ptr = Rc::into_raw(clone) as *const ();
131        RawWaker::new(ptr, &VTABLE)
132    }
133}
134
135unsafe fn wake_task(data: *const ()) {
136    // SAFETY: VTABLE supplies an owned task reference created by Rc::into_raw.
137    // Reconstruct it so the consumed waker's reference is released on return
138    // or unwind. The queue receives a separate cloned reference.
139    unsafe {
140        let rc_task = Rc::from_raw(data as *const Task);
141        let cloned = rc_task.clone();
142        rc_task.executor_state.new_tasks.borrow_mut().push(cloned);
143    }
144}
145
146struct Task {
147    future: RefCell<Option<Pin<Box<dyn Future<Output = SimResult>>>>>,
148    executor_state: Rc<ExecutorState>,
149}
150
151impl Task {
152    pub fn new(
153        future: impl Future<Output = SimResult> + 'static,
154        executor_state: Rc<ExecutorState>,
155    ) -> Task {
156        Task {
157            future: RefCell::new(Some(Box::pin(future))),
158            executor_state,
159        }
160    }
161
162    fn poll(&self, context: &mut Context) -> Option<Poll<SimResult>> {
163        let mut future_slot = self.future.borrow_mut();
164        let future = future_slot.as_mut()?;
165
166        let poll_result = future.as_mut().poll(context);
167        if poll_result.is_ready() {
168            future_slot.take();
169        }
170
171        Some(poll_result)
172    }
173}
174
175/// Cumulative future activity from the most recent executor run.
176#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
177pub struct ExecutorStats {
178    /// Number of calls made to an async future's `poll` method.
179    pub futures_polled: usize,
180    /// Number of async futures that returned `Poll::Ready`.
181    pub futures_completed: usize,
182}
183
184/// Executor activity reported at a configured wall-clock interval.
185#[derive(Clone, Copy, Debug, Eq, PartialEq)]
186pub struct ExecutorSnapshot {
187    /// Cumulative future activity so far in this run.
188    pub stats: ExecutorStats,
189    /// Number of tasks currently active in the executor.
190    pub active_task_count: usize,
191}
192
193struct ExecutorState {
194    new_tasks: RefCell<Vec<Rc<Task>>>,
195    time: RefCell<SimTime>,
196    randomize_task_order: Cell<bool>,
197    task_order_rng: RefCell<StdRng>,
198    stats: Cell<ExecutorStats>,
199}
200
201impl ExecutorState {
202    pub fn new(top: &Rc<Entity>) -> Self {
203        Self {
204            new_tasks: RefCell::new(Vec::new()),
205            time: RefCell::new(SimTime::new(top)),
206            randomize_task_order: Cell::new(false),
207            task_order_rng: RefCell::new(StdRng::seed_from_u64(rand::random())),
208            stats: Cell::new(ExecutorStats::default()),
209        }
210    }
211}
212
213/// Single-threaded executor
214///
215/// This is a thin-wrapper (using [`Rc`]) around the real executor, so that this
216/// struct can be cloned and passed around.
217///
218/// See the [module documentation] for more details.
219///
220/// [module documentation]: index.html
221#[derive(Clone)]
222pub struct Executor {
223    state: Rc<ExecutorState>,
224}
225
226impl Executor {
227    /// Run the executor without observing activity.
228    pub(crate) fn run(&self) -> SimResult {
229        self.run_internal(None, |_| Ok(()))
230    }
231
232    /// Run the executor while reporting activity after each wall-clock
233    /// interval has passed. Observations occur at completed step boundaries.
234    pub(crate) fn run_with_observer(
235        &self,
236        minimum_interval: Duration,
237        observer: impl FnMut(ExecutorSnapshot) -> SimResult,
238    ) -> SimResult {
239        if minimum_interval.is_zero() {
240            return Err(crate::types::SimError(
241                "executor observer interval must be positive".to_string(),
242            ));
243        }
244
245        let refresh_due = Arc::new(AtomicBool::new(false));
246        let timer_refresh_due = Arc::clone(&refresh_due);
247        let (stop_timer, timer_stop) = mpsc::channel::<()>();
248        let timer = thread::spawn(move || {
249            // Use the timeout mechanism on the channel to give us time
250            // intervals. Will break out when the main thread
251            // disconnects the channel.
252            while let Err(RecvTimeoutError::Timeout) = timer_stop.recv_timeout(minimum_interval) {
253                timer_refresh_due.store(true, Ordering::Relaxed);
254            }
255        });
256
257        let result = self.run_internal(Some(&refresh_due), observer);
258        drop(stop_timer);
259        timer
260            .join()
261            .expect("executor observer timer thread panicked");
262        result
263    }
264
265    fn run_internal(
266        &self,
267        refresh_due: Option<&AtomicBool>,
268        mut observer: impl FnMut(ExecutorSnapshot) -> SimResult,
269    ) -> SimResult {
270        let mut stats = ExecutorStats::default();
271
272        // Reuse the polling buffer across steps. Start with room for 100 tasks
273        // to avoid allocating for typical batches; larger batches grow it.
274        let mut task_queue = Vec::with_capacity(100);
275
276        let result = (|| {
277            loop {
278                self.step(&mut task_queue, &mut stats)?;
279
280                if refresh_due.is_some_and(|due| {
281                    due.load(Ordering::Relaxed) && due.swap(false, Ordering::Relaxed)
282                }) {
283                    observer(ExecutorSnapshot {
284                        stats,
285                        active_task_count: self.state.new_tasks.borrow().len(),
286                    })?;
287                }
288
289                if self.state.new_tasks.borrow().is_empty() {
290                    if self.state.time.borrow().can_exit() {
291                        break;
292                    }
293
294                    if let Some(wakers) = self.state.time.borrow_mut().advance_time() {
295                        // No events left, advance time
296                        for task_waker in wakers.into_iter() {
297                            task_waker.waker.wake();
298                        }
299                    } else {
300                        break;
301                    }
302                }
303            }
304            Ok(())
305        })();
306        self.state.stats.set(stats);
307        result
308    }
309
310    // Run a simulation step.
311    //
312    // The caller passes in task_queue as scratch buffer so its allocation can
313    // be reused by each call to step. It is fully drained before step
314    // completes unless there is an error.
315    fn step(&self, task_queue: &mut Vec<Rc<Task>>, stats: &mut ExecutorStats) -> SimResult {
316        // Append new tasks created since the last step into the task queue
317        task_queue.append(&mut self.state.new_tasks.borrow_mut());
318        if self.state.randomize_task_order.get() {
319            task_queue.shuffle(&mut *self.state.task_order_rng.borrow_mut());
320        }
321
322        // Poll each queued task once. Pending tasks must arrange to be queued
323        // again through a saved waker; they are not retained here.
324        for task in task_queue.drain(..) {
325            let waker = waker_for_task(&task);
326            let mut context = Context::from_waker(&waker);
327
328            if let Some(poll_result) = task.poll(&mut context) {
329                stats.futures_polled += 1;
330                match poll_result {
331                    Poll::Ready(Err(e)) => {
332                        stats.futures_completed += 1;
333                        return Err(e);
334                    }
335                    Poll::Ready(Ok(())) => {
336                        stats.futures_completed += 1;
337                    }
338                    Poll::Pending => {}
339                }
340            }
341        }
342        Ok(())
343    }
344
345    /// Return cumulative future activity from the most recent executor run.
346    #[must_use]
347    pub fn stats(&self) -> ExecutorStats {
348        self.state.stats.get()
349    }
350
351    #[must_use]
352    pub fn get_clock(&self, freq_mhz: f64) -> Clock {
353        self.state.time.borrow_mut().get_clock(freq_mhz)
354    }
355
356    #[must_use]
357    pub fn time_now_ns(&self) -> f64 {
358        self.state.time.borrow().time_now_ns()
359    }
360
361    pub fn set_randomize_task_order(&self, randomize: bool) {
362        self.state.randomize_task_order.set(randomize);
363    }
364
365    pub fn set_task_order_seed(&self, seed: u64) {
366        *self.state.task_order_rng.borrow_mut() = StdRng::seed_from_u64(seed);
367    }
368}
369
370/// `Spawner` spawns new futures into the executor.
371#[derive(Clone)]
372pub struct Spawner {
373    state: Rc<ExecutorState>,
374}
375
376impl Spawner {
377    pub fn spawn(&self, future: impl Future<Output = SimResult> + 'static) {
378        self.state
379            .new_tasks
380            .borrow_mut()
381            .push(Rc::new(Task::new(future, self.state.clone())));
382    }
383}
384
385#[must_use]
386pub fn new_executor_and_spawner(top: &Rc<Entity>) -> (Executor, Spawner) {
387    let state = Rc::new(ExecutorState::new(top));
388    (
389        Executor {
390            state: state.clone(),
391        },
392        Spawner { state },
393    )
394}
395
396#[cfg(test)]
397mod tests {
398    use std::future::Future;
399    use std::pin::Pin;
400    use std::rc::Rc;
401    use std::sync::atomic::{AtomicBool, Ordering};
402    use std::task::{Context, Poll};
403    use std::time::Duration;
404
405    use futures::task::noop_waker;
406    use gwr_track::entity::toplevel;
407    use gwr_track::tracker::dev_null_tracker;
408
409    use super::*;
410    use crate::time::clock::TaskWaker;
411
412    fn new_executor(top: &Rc<Entity>) -> Executor {
413        Executor {
414            state: Rc::new(ExecutorState::new(top)),
415        }
416    }
417
418    struct PendingOnce {
419        was_polled: bool,
420    }
421
422    impl Future for PendingOnce {
423        type Output = SimResult;
424
425        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
426            if self.was_polled {
427                Poll::Ready(Ok(()))
428            } else {
429                self.was_polled = true;
430                #[expect(clippy::waker_clone_wake)]
431                cx.waker().clone().wake();
432                Poll::Pending
433            }
434        }
435    }
436
437    #[test]
438    fn wake_by_ref_queues_task_without_consuming_the_waker() {
439        for use_owned_waker in [false, true] {
440            let tracker = dev_null_tracker();
441            let top = toplevel(&tracker, "top");
442            let executor = new_executor(&top);
443            let completed = Rc::new(Cell::new(false));
444            let completion = completed.clone();
445            let task = Rc::new(Task::new(
446                async move {
447                    completion.set(true);
448                    Ok(())
449                },
450                executor.state.clone(),
451            ));
452            let borrowed = waker_for_task(&task);
453            let owned = use_owned_waker.then(|| Waker::clone(&borrowed));
454            let waker = owned.as_ref().unwrap_or(&borrowed);
455            let references = Rc::strong_count(&task);
456
457            waker.wake_by_ref();
458            waker.wake_by_ref();
459            assert_eq!(Rc::strong_count(&task), references + 2);
460
461            let mut stats = ExecutorStats::default();
462            executor.step(&mut Vec::new(), &mut stats).unwrap();
463            assert!(completed.get());
464            assert_eq!(stats.futures_polled, 1);
465            assert_eq!(Rc::strong_count(&task), references);
466            drop(owned);
467            assert_eq!(Rc::strong_count(&task), 1);
468        }
469    }
470
471    #[test]
472    fn poll_context_does_not_hold_an_extra_task_reference() {
473        let tracker = dev_null_tracker();
474        let top = toplevel(&tracker, "top");
475        let executor = new_executor(&top);
476        let observed_references = Rc::new(Cell::new(0));
477        let task: Rc<Task> = Rc::new_cyclic(|weak| {
478            let weak = weak.clone();
479            let observed = observed_references.clone();
480            Task::new(
481                async move {
482                    observed.set(weak.strong_count());
483                    Ok(())
484                },
485                executor.state.clone(),
486            )
487        });
488        executor.state.new_tasks.borrow_mut().push(task.clone());
489
490        let mut task_queue = Vec::new();
491        executor
492            .step(&mut task_queue, &mut ExecutorStats::default())
493            .unwrap();
494
495        // The test handle and the executor's task handle
496        assert_eq!(observed_references.get(), 2);
497        assert_eq!(Rc::strong_count(&task), 1);
498    }
499
500    #[test]
501    fn cloned_context_waker_keeps_task_alive_after_poll() {
502        let tracker = dev_null_tracker();
503        let top = toplevel(&tracker, "top");
504        let executor = new_executor(&top);
505        let saved_waker = Rc::new(RefCell::new(None));
506        let saved = saved_waker.clone();
507        let mut was_polled = false;
508        let future = std::future::poll_fn(move |cx| {
509            if was_polled {
510                Poll::Ready(Ok(()))
511            } else {
512                was_polled = true;
513                *saved.borrow_mut() = Some(cx.waker().clone());
514                Poll::Pending
515            }
516        });
517        let task = Rc::new(Task::new(future, executor.state.clone()));
518        let weak = Rc::downgrade(&task);
519        executor.state.new_tasks.borrow_mut().push(task);
520        let mut stats = ExecutorStats::default();
521        let mut task_queue = Vec::new();
522
523        executor.step(&mut task_queue, &mut stats).unwrap();
524        assert_eq!(weak.strong_count(), 1);
525        assert!(executor.state.new_tasks.borrow().is_empty());
526
527        saved_waker.borrow_mut().take().unwrap().wake();
528        assert_eq!(executor.state.new_tasks.borrow().len(), 1);
529        executor.step(&mut task_queue, &mut stats).unwrap();
530
531        assert_eq!(weak.strong_count(), 0);
532        assert_eq!(stats.futures_polled, 2);
533        assert_eq!(stats.futures_completed, 1);
534    }
535
536    #[test]
537    fn owned_waker_clone_survives_borrowed_context_and_original_waker() {
538        let tracker = dev_null_tracker();
539        let top = toplevel(&tracker, "top");
540        let executor = new_executor(&top);
541        let task = Rc::new(Task::new(async { Ok(()) }, executor.state.clone()));
542        let weak = Rc::downgrade(&task);
543        let owned = {
544            let borrowed = waker_for_task(&task);
545            Waker::clone(&borrowed)
546        };
547        drop(task);
548
549        let cloned = owned.clone();
550        assert_eq!(weak.strong_count(), 2);
551        drop(owned);
552        assert_eq!(weak.strong_count(), 1);
553
554        cloned.wake();
555        assert_eq!(executor.state.new_tasks.borrow().len(), 1);
556        let mut task_queue = Vec::new();
557        executor
558            .step(&mut task_queue, &mut ExecutorStats::default())
559            .unwrap();
560        assert_eq!(weak.strong_count(), 0);
561    }
562
563    #[test]
564    fn wake_releases_its_task_reference_when_queue_borrowing_panics() {
565        let tracker = dev_null_tracker();
566        let top = toplevel(&tracker, "top");
567        let executor = new_executor(&top);
568        let task = Rc::new(Task::new(async { Ok(()) }, executor.state.clone()));
569        let weak = Rc::downgrade(&task);
570        let waker = Waker::clone(&waker_for_task(&task));
571        drop(task);
572        let queue = executor.state.new_tasks.borrow_mut();
573
574        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| waker.wake()));
575
576        assert!(result.is_err());
577        assert!(queue.is_empty());
578        assert_eq!(weak.strong_count(), 0);
579    }
580
581    #[test]
582    fn poll_panic_does_not_consume_the_callers_task_reference() {
583        let tracker = dev_null_tracker();
584        let top = toplevel(&tracker, "top");
585        let executor = new_executor(&top);
586        let task = Rc::new(Task::new(
587            async { panic!("poll failed") },
588            executor.state.clone(),
589        ));
590        executor.state.new_tasks.borrow_mut().push(task.clone());
591        let mut task_queue = Vec::new();
592
593        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
594            executor.step(&mut task_queue, &mut ExecutorStats::default())
595        }));
596
597        assert!(result.is_err());
598        assert!(task_queue.is_empty());
599        assert_eq!(Rc::strong_count(&task), 1);
600    }
601
602    #[test]
603    fn run_exits_when_time_cannot_advance() {
604        let tracker = dev_null_tracker();
605        let top = toplevel(&tracker, "top");
606        let executor = new_executor(&top);
607        let clock = executor.get_clock(1000.0);
608
609        clock
610            .shared_state
611            .waiting
612            .borrow_mut()
613            .push(vec![TaskWaker {
614                id: 0,
615                waker: noop_waker(),
616                can_exit: false,
617            }]);
618
619        executor.run().unwrap();
620    }
621
622    #[test]
623    fn executor_observer_reports_once_when_refresh_is_due() {
624        let tracker = dev_null_tracker();
625        let top = toplevel(&tracker, "top");
626        let (executor, spawner) = new_executor_and_spawner(&top);
627
628        for _ in 0..5 {
629            spawner.spawn(async { Ok(()) });
630        }
631
632        let mut snapshots = Vec::new();
633        let refresh_due = AtomicBool::new(true);
634        executor
635            .run_internal(Some(&refresh_due), |snapshot| {
636                snapshots.push(snapshot);
637                Ok(())
638            })
639            .unwrap();
640
641        assert!(!refresh_due.load(Ordering::Relaxed));
642
643        assert_eq!(
644            snapshots,
645            vec![ExecutorSnapshot {
646                stats: ExecutorStats {
647                    futures_polled: 5,
648                    futures_completed: 5,
649                },
650                active_task_count: 0,
651            }]
652        );
653        assert_eq!(
654            executor.stats(),
655            ExecutorStats {
656                futures_polled: 5,
657                futures_completed: 5,
658            }
659        );
660    }
661
662    #[test]
663    fn executor_observer_skips_steps_without_a_refresh_signal() {
664        let tracker = dev_null_tracker();
665        let top = toplevel(&tracker, "top");
666        let (executor, spawner) = new_executor_and_spawner(&top);
667        spawner.spawn(async { Ok(()) });
668
669        let refresh_due = AtomicBool::new(false);
670        let mut snapshots = Vec::new();
671        executor
672            .run_internal(Some(&refresh_due), |snapshot| {
673                snapshots.push(snapshot);
674                Ok(())
675            })
676            .unwrap();
677
678        assert!(snapshots.is_empty());
679    }
680
681    #[test]
682    fn executor_stats_count_pending_future_polls_separately_from_completion() {
683        let tracker = dev_null_tracker();
684        let top = toplevel(&tracker, "top");
685        let (executor, spawner) = new_executor_and_spawner(&top);
686        spawner.spawn(PendingOnce { was_polled: false });
687
688        executor.run().unwrap();
689
690        assert_eq!(
691            executor.stats(),
692            ExecutorStats {
693                futures_polled: 2,
694                futures_completed: 1,
695            }
696        );
697    }
698
699    #[test]
700    fn executor_stats_include_futures_that_complete_with_an_error() {
701        let tracker = dev_null_tracker();
702        let top = toplevel(&tracker, "top");
703        let (executor, spawner) = new_executor_and_spawner(&top);
704        spawner.spawn(async { Err(crate::types::SimError("failed".to_string())) });
705
706        let result = executor.run();
707
708        assert!(result.is_err());
709        assert_eq!(
710            executor.stats(),
711            ExecutorStats {
712                futures_polled: 1,
713                futures_completed: 1,
714            }
715        );
716    }
717
718    #[test]
719    fn executor_observer_is_not_called_from_a_step_that_returns_a_task_error() {
720        let tracker = dev_null_tracker();
721        let top = toplevel(&tracker, "top");
722        let (executor, spawner) = new_executor_and_spawner(&top);
723        spawner.spawn(async { Err(crate::types::SimError("task failed".to_string())) });
724
725        let mut snapshots = Vec::new();
726        let refresh_due = AtomicBool::new(true);
727        let error = executor
728            .run_internal(Some(&refresh_due), |snapshot| {
729                snapshots.push(snapshot);
730                Err(crate::types::SimError("observer failed".to_string()))
731            })
732            .unwrap_err();
733
734        assert_eq!(error.to_string(), "task failed");
735        assert!(snapshots.is_empty());
736        assert_eq!(
737            executor.stats(),
738            ExecutorStats {
739                futures_polled: 1,
740                futures_completed: 1,
741            }
742        );
743    }
744
745    #[test]
746    fn observer_error_occurs_after_the_entire_step_batch() {
747        let tracker = dev_null_tracker();
748        let top = toplevel(&tracker, "top");
749        let (executor, spawner) = new_executor_and_spawner(&top);
750        let completions = Rc::new(Cell::new(0));
751        for _ in 0..3 {
752            let completions = completions.clone();
753            spawner.spawn(async move {
754                completions.set(completions.get() + 1);
755                Ok(())
756            });
757        }
758
759        let refresh_due = AtomicBool::new(true);
760        let first_error = executor
761            .run_internal(Some(&refresh_due), |_| {
762                Err(crate::types::SimError("observer failed".to_string()))
763            })
764            .unwrap_err();
765        assert_eq!(first_error.to_string(), "observer failed");
766        assert_eq!(completions.get(), 3);
767        assert_eq!(
768            executor.stats(),
769            ExecutorStats {
770                futures_polled: 3,
771                futures_completed: 3,
772            }
773        );
774    }
775
776    #[test]
777    fn executor_observer_rejects_a_zero_wall_clock_interval() {
778        let tracker = dev_null_tracker();
779        let top = toplevel(&tracker, "top");
780        let executor = new_executor(&top);
781        let error = executor
782            .run_with_observer(Duration::ZERO, |_| Ok(()))
783            .unwrap_err();
784
785        assert_eq!(
786            error.to_string(),
787            "executor observer interval must be positive"
788        );
789    }
790}