Skip to main content

mobench_provider/
execution.rs

1//! Provider execution Interface and Module-owned lifecycle policy.
2
3use std::collections::HashMap;
4use std::error::Error;
5use std::num::NonZeroU8;
6
7use mobench_process::ProcessCancellation;
8use thiserror::Error;
9
10use crate::{
11    CollectedSession, ExpectedSession, MatrixAssessment, ReconcileError, reconcile_sessions,
12};
13
14/// One collected Provider Session with its report payloads.
15#[derive(Clone, Debug, PartialEq, Eq)]
16pub struct CollectedOutput<Report> {
17    /// Provider session identifier used for correlation.
18    pub session_id: String,
19    /// Reports attributed to this session before cardinality validation.
20    pub reports: Vec<Report>,
21    /// Optional already-sanitized failure summary from the Adapter.
22    pub failure: Option<String>,
23}
24
25/// Terminal evidence returned by an Adapter before reconciliation.
26#[derive(Clone, Debug, PartialEq, Eq)]
27pub struct AdapterRun<Report> {
28    /// Terminal Provider Sessions expected for the requested matrix.
29    pub expected: Vec<ExpectedSession>,
30    /// Collected reports and failures, in arbitrary Adapter order.
31    pub collected: Vec<CollectedOutput<Report>>,
32}
33
34impl<Report> AdapterRun<Report> {
35    /// Reconcile terminal Adapter evidence into deterministic Provider receipts.
36    pub fn reconcile(self) -> Result<ProviderRun<Report>, ReconcileError> {
37        let summaries = self
38            .collected
39            .iter()
40            .map(|session| CollectedSession {
41                session_id: session.session_id.clone(),
42                report_count: session.reports.len(),
43                failure: session.failure.clone(),
44            })
45            .collect::<Vec<_>>();
46        let assessment = reconcile_sessions(&self.expected, &summaries)?;
47
48        let mut collected_by_session = self
49            .collected
50            .into_iter()
51            .map(|session| (session.session_id.clone(), session))
52            .collect::<HashMap<_, _>>();
53        let sessions = self
54            .expected
55            .iter()
56            .filter_map(|expected| collected_by_session.remove(&expected.session_id))
57            .collect();
58
59        Ok(ProviderRun {
60            assessment,
61            sessions,
62        })
63    }
64}
65
66/// Reconciled Provider Run with deterministic session ordering.
67#[derive(Clone, Debug, PartialEq, Eq)]
68pub struct ProviderRun<Report> {
69    assessment: MatrixAssessment,
70    sessions: Vec<CollectedOutput<Report>>,
71}
72
73impl<Report> ProviderRun<Report> {
74    /// Complete/partial/failed assessment for the entire Provider Run.
75    pub const fn assessment(&self) -> &MatrixAssessment {
76        &self.assessment
77    }
78
79    /// Collected Provider Sessions in expected-matrix order.
80    ///
81    /// Missing sessions have no entry here and remain visible in
82    /// [`Self::assessment`].
83    pub fn sessions(&self) -> &[CollectedOutput<Report>] {
84        &self.sessions
85    }
86
87    /// Consume the receipt and collected payloads.
88    pub fn into_parts(self) -> (MatrixAssessment, Vec<CollectedOutput<Report>>) {
89        (self.assessment, self.sessions)
90    }
91}
92
93/// Adapter Interface at the provider Seam.
94///
95/// The Adapter owns provider-specific transport, credentials, artifact upload,
96/// polling, and report extraction. [`ProviderEngine`] owns call ordering,
97/// cooperative cancellation, collection retry policy, and reconciliation.
98pub trait ProviderAdapter {
99    /// Resolved request consumed by this Adapter.
100    type Request;
101    /// Durable handle returned after the Provider Run starts.
102    type Handle;
103    /// One collected report payload.
104    type Report;
105    /// Provider-specific failure.
106    type Error: Error + Send + Sync + 'static;
107
108    /// Start a Provider Run without waiting for terminal reports.
109    fn start(
110        &self,
111        request: &Self::Request,
112        cancellation: &ProcessCancellation,
113    ) -> Result<Self::Handle, Self::Error>;
114
115    /// Collect terminal session evidence for a previously started run.
116    fn collect(
117        &self,
118        handle: &Self::Handle,
119        cancellation: &ProcessCancellation,
120    ) -> Result<AdapterRun<Self::Report>, Self::Error>;
121
122    /// Best-effort cancellation of a started Provider Run.
123    fn cancel(
124        &self,
125        handle: &Self::Handle,
126        cancellation: &ProcessCancellation,
127    ) -> Result<(), Self::Error>;
128
129    /// Whether collection can safely be retried after this failure.
130    fn is_collect_retryable(&self, _error: &Self::Error) -> bool {
131        false
132    }
133}
134
135/// A Provider Run that has started but has not yet been collected.
136#[derive(Clone, Debug, PartialEq, Eq)]
137pub struct StartedRun<Handle> {
138    handle: Handle,
139}
140
141impl<Handle> StartedRun<Handle> {
142    /// Resume collection from an Adapter-specific durable handle.
143    pub const fn from_handle(handle: Handle) -> Self {
144        Self { handle }
145    }
146
147    /// Borrow the Adapter-specific durable handle.
148    pub const fn handle(&self) -> &Handle {
149        &self.handle
150    }
151
152    /// Consume the typestate and return its durable handle.
153    pub fn into_handle(self) -> Handle {
154        self.handle
155    }
156}
157
158/// Provider lifecycle stage attached to execution failures.
159#[derive(Clone, Copy, Debug, PartialEq, Eq)]
160pub enum ExecutionStage {
161    /// Before or while starting the Provider Run.
162    Start,
163    /// While collecting terminal evidence.
164    Collect,
165    /// While cancelling a started Provider Run.
166    Cancel,
167}
168
169/// Provider execution failure with stable lifecycle context.
170#[derive(Debug, Error)]
171pub enum ProviderExecutionError<AdapterError>
172where
173    AdapterError: Error + 'static,
174{
175    /// Cooperative cancellation was requested.
176    #[error("provider execution cancelled during {stage:?}")]
177    Cancelled { stage: ExecutionStage },
178    /// The Adapter failed during one lifecycle stage.
179    #[error("provider adapter failed during {stage:?} after {attempts} attempt(s): {source}")]
180    Adapter {
181        /// Lifecycle stage that failed.
182        stage: ExecutionStage,
183        /// Number of calls made at this stage.
184        attempts: u8,
185        /// Adapter-specific source error.
186        #[source]
187        source: AdapterError,
188    },
189    /// Cancellation was requested but the Adapter could not cancel the run.
190    #[error("provider adapter failed to cancel a started run: {source}")]
191    Cancellation {
192        /// Adapter-specific source error.
193        #[source]
194        source: AdapterError,
195    },
196    /// Terminal evidence was structurally ambiguous.
197    #[error("provider session reconciliation failed: {0}")]
198    Reconcile(#[from] ReconcileError),
199}
200
201/// Module-owned provider lifecycle with bounded collection retries.
202#[derive(Clone, Debug)]
203pub struct ProviderEngine<Adapter> {
204    adapter: Adapter,
205    max_collect_attempts: NonZeroU8,
206}
207
208impl<Adapter> ProviderEngine<Adapter>
209where
210    Adapter: ProviderAdapter,
211{
212    /// Construct an engine with one collection attempt.
213    pub fn new(adapter: Adapter) -> Self {
214        Self {
215            adapter,
216            max_collect_attempts: NonZeroU8::MIN,
217        }
218    }
219
220    /// Set the maximum number of collection attempts.
221    ///
222    /// Start is never retried because remote scheduling is not assumed to be
223    /// idempotent. Only failures explicitly marked retryable by the Adapter are
224    /// retried.
225    pub fn with_max_collect_attempts(adapter: Adapter, max_collect_attempts: NonZeroU8) -> Self {
226        Self {
227            adapter,
228            max_collect_attempts,
229        }
230    }
231
232    /// Borrow the concrete Adapter.
233    pub const fn adapter(&self) -> &Adapter {
234        &self.adapter
235    }
236
237    /// Start a Provider Run.
238    pub fn start(
239        &self,
240        request: &Adapter::Request,
241        cancellation: &ProcessCancellation,
242    ) -> Result<StartedRun<Adapter::Handle>, ProviderExecutionError<Adapter::Error>> {
243        if cancellation.is_cancelled() {
244            return Err(ProviderExecutionError::Cancelled {
245                stage: ExecutionStage::Start,
246            });
247        }
248        let handle = self
249            .adapter
250            .start(request, cancellation)
251            .map_err(|source| ProviderExecutionError::Adapter {
252                stage: ExecutionStage::Start,
253                attempts: 1,
254                source,
255            })?;
256        Ok(StartedRun { handle })
257    }
258
259    /// Collect and reconcile a started Provider Run.
260    pub fn collect(
261        &self,
262        started: StartedRun<Adapter::Handle>,
263        cancellation: &ProcessCancellation,
264    ) -> Result<ProviderRun<Adapter::Report>, ProviderExecutionError<Adapter::Error>> {
265        let mut attempts = 0u8;
266        loop {
267            if cancellation.is_cancelled() {
268                self.adapter
269                    .cancel(&started.handle, cancellation)
270                    .map_err(|source| ProviderExecutionError::Cancellation { source })?;
271                return Err(ProviderExecutionError::Cancelled {
272                    stage: ExecutionStage::Collect,
273                });
274            }
275
276            attempts = attempts.saturating_add(1);
277            match self.adapter.collect(&started.handle, cancellation) {
278                Ok(run) => return run.reconcile().map_err(Into::into),
279                Err(source) => {
280                    if cancellation.is_cancelled() {
281                        self.adapter
282                            .cancel(&started.handle, cancellation)
283                            .map_err(|source| ProviderExecutionError::Cancellation { source })?;
284                        return Err(ProviderExecutionError::Cancelled {
285                            stage: ExecutionStage::Collect,
286                        });
287                    }
288                    let retryable = self.adapter.is_collect_retryable(&source);
289                    if !retryable || attempts >= self.max_collect_attempts.get() {
290                        return Err(ProviderExecutionError::Adapter {
291                            stage: ExecutionStage::Collect,
292                            attempts,
293                            source,
294                        });
295                    }
296                }
297            }
298        }
299    }
300
301    /// Start, collect, and reconcile one Provider Run.
302    pub fn execute(
303        &self,
304        request: &Adapter::Request,
305        cancellation: &ProcessCancellation,
306    ) -> Result<ProviderRun<Adapter::Report>, ProviderExecutionError<Adapter::Error>> {
307        let started = self.start(request, cancellation)?;
308        self.collect(started, cancellation)
309    }
310
311    /// Cancel a started Provider Run.
312    pub fn cancel(
313        &self,
314        started: StartedRun<Adapter::Handle>,
315        cancellation: &ProcessCancellation,
316    ) -> Result<(), ProviderExecutionError<Adapter::Error>> {
317        self.adapter
318            .cancel(&started.handle, cancellation)
319            .map_err(|source| ProviderExecutionError::Adapter {
320                stage: ExecutionStage::Cancel,
321                attempts: 1,
322                source,
323            })
324    }
325}
326
327#[cfg(test)]
328mod tests {
329    use std::cell::Cell;
330    use std::fmt;
331
332    use super::*;
333    use crate::{MatrixOutcome, SessionDisposition};
334
335    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
336    enum FakeFailure {
337        Retryable,
338        Permanent,
339        Cancel,
340    }
341
342    impl fmt::Display for FakeFailure {
343        fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
344            write!(formatter, "{self:?}")
345        }
346    }
347
348    impl Error for FakeFailure {}
349
350    struct FakeAdapter {
351        start_calls: Cell<u8>,
352        collect_calls: Cell<u8>,
353        cancel_calls: Cell<u8>,
354        retryable_failures: Cell<u8>,
355        permanent_failure: bool,
356        cancel_failure: bool,
357        unexpected_session: bool,
358    }
359
360    impl FakeAdapter {
361        fn successful() -> Self {
362            Self {
363                start_calls: Cell::new(0),
364                collect_calls: Cell::new(0),
365                cancel_calls: Cell::new(0),
366                retryable_failures: Cell::new(0),
367                permanent_failure: false,
368                cancel_failure: false,
369                unexpected_session: false,
370            }
371        }
372
373        fn terminal_run(&self) -> AdapterRun<u64> {
374            let expected = vec![
375                ExpectedSession {
376                    session_id: "session-a".to_owned(),
377                    device_id: "device-a".to_owned(),
378                    status: "passed".to_owned(),
379                },
380                ExpectedSession {
381                    session_id: "session-b".to_owned(),
382                    device_id: "device-b".to_owned(),
383                    status: "passed".to_owned(),
384                },
385            ];
386            let mut collected = vec![
387                CollectedOutput {
388                    session_id: "session-b".to_owned(),
389                    reports: vec![22],
390                    failure: None,
391                },
392                CollectedOutput {
393                    session_id: "session-a".to_owned(),
394                    reports: vec![11],
395                    failure: None,
396                },
397            ];
398            if self.unexpected_session {
399                collected.push(CollectedOutput {
400                    session_id: "unexpected".to_owned(),
401                    reports: vec![33],
402                    failure: None,
403                });
404            }
405            AdapterRun {
406                expected,
407                collected,
408            }
409        }
410    }
411
412    impl ProviderAdapter for FakeAdapter {
413        type Request = ();
414        type Handle = &'static str;
415        type Report = u64;
416        type Error = FakeFailure;
417
418        fn start(
419            &self,
420            _request: &Self::Request,
421            _cancellation: &ProcessCancellation,
422        ) -> Result<Self::Handle, Self::Error> {
423            self.start_calls.set(self.start_calls.get() + 1);
424            Ok("run-1")
425        }
426
427        fn collect(
428            &self,
429            _handle: &Self::Handle,
430            _cancellation: &ProcessCancellation,
431        ) -> Result<AdapterRun<Self::Report>, Self::Error> {
432            self.collect_calls.set(self.collect_calls.get() + 1);
433            if self.permanent_failure {
434                return Err(FakeFailure::Permanent);
435            }
436            if self.retryable_failures.get() > 0 {
437                self.retryable_failures
438                    .set(self.retryable_failures.get() - 1);
439                return Err(FakeFailure::Retryable);
440            }
441            Ok(self.terminal_run())
442        }
443
444        fn cancel(
445            &self,
446            _handle: &Self::Handle,
447            _cancellation: &ProcessCancellation,
448        ) -> Result<(), Self::Error> {
449            self.cancel_calls.set(self.cancel_calls.get() + 1);
450            if self.cancel_failure {
451                Err(FakeFailure::Cancel)
452            } else {
453                Ok(())
454            }
455        }
456
457        fn is_collect_retryable(&self, error: &Self::Error) -> bool {
458            matches!(error, FakeFailure::Retryable)
459        }
460    }
461
462    #[test]
463    fn execute_orders_collected_payloads_by_expected_matrix() {
464        let adapter = FakeAdapter::successful();
465        let engine = ProviderEngine::new(adapter);
466        let run = engine
467            .execute(&(), &ProcessCancellation::default())
468            .unwrap();
469
470        assert_eq!(run.assessment().outcome(), MatrixOutcome::Complete);
471        assert_eq!(run.sessions()[0].session_id, "session-a");
472        assert_eq!(run.sessions()[0].reports, vec![11]);
473        assert_eq!(run.sessions()[1].session_id, "session-b");
474        assert_eq!(run.sessions()[1].reports, vec![22]);
475        assert_eq!(engine.adapter().start_calls.get(), 1);
476        assert_eq!(engine.adapter().collect_calls.get(), 1);
477    }
478
479    #[test]
480    fn explicit_start_then_collect_uses_the_same_handle() {
481        let engine = ProviderEngine::new(FakeAdapter::successful());
482        let cancellation = ProcessCancellation::default();
483        let started = engine.start(&(), &cancellation).unwrap();
484        assert_eq!(*started.handle(), "run-1");
485        let run = engine.collect(started, &cancellation).unwrap();
486        assert!(run.assessment().is_complete());
487    }
488
489    #[test]
490    fn retryable_collection_failure_is_bounded() {
491        let adapter = FakeAdapter::successful();
492        adapter.retryable_failures.set(2);
493        let engine = ProviderEngine::with_max_collect_attempts(adapter, NonZeroU8::new(3).unwrap());
494        let run = engine
495            .execute(&(), &ProcessCancellation::default())
496            .unwrap();
497        assert!(run.assessment().is_complete());
498        assert_eq!(engine.adapter().collect_calls.get(), 3);
499    }
500
501    #[test]
502    fn retryable_collection_failure_stops_at_the_limit() {
503        let adapter = FakeAdapter::successful();
504        adapter.retryable_failures.set(3);
505        let engine = ProviderEngine::with_max_collect_attempts(adapter, NonZeroU8::new(2).unwrap());
506        let error = engine
507            .execute(&(), &ProcessCancellation::default())
508            .unwrap_err();
509        assert!(matches!(
510            error,
511            ProviderExecutionError::Adapter {
512                stage: ExecutionStage::Collect,
513                attempts: 2,
514                source: FakeFailure::Retryable,
515            }
516        ));
517    }
518
519    #[test]
520    fn permanent_collection_failure_is_not_retried() {
521        let mut adapter = FakeAdapter::successful();
522        adapter.permanent_failure = true;
523        let engine = ProviderEngine::with_max_collect_attempts(adapter, NonZeroU8::new(5).unwrap());
524        let error = engine
525            .execute(&(), &ProcessCancellation::default())
526            .unwrap_err();
527        assert!(matches!(
528            error,
529            ProviderExecutionError::Adapter {
530                stage: ExecutionStage::Collect,
531                attempts: 1,
532                source: FakeFailure::Permanent,
533            }
534        ));
535    }
536
537    #[test]
538    fn cancellation_before_start_never_calls_the_adapter() {
539        let cancellation = ProcessCancellation::default();
540        cancellation.cancel();
541        let engine = ProviderEngine::new(FakeAdapter::successful());
542        let error = engine.start(&(), &cancellation).unwrap_err();
543        assert!(matches!(
544            error,
545            ProviderExecutionError::Cancelled {
546                stage: ExecutionStage::Start
547            }
548        ));
549        assert_eq!(engine.adapter().start_calls.get(), 0);
550    }
551
552    #[test]
553    fn cancellation_after_start_cancels_the_adapter_before_returning() {
554        let cancellation = ProcessCancellation::default();
555        let engine = ProviderEngine::new(FakeAdapter::successful());
556        let started = engine.start(&(), &cancellation).unwrap();
557        cancellation.cancel();
558        let error = engine.collect(started, &cancellation).unwrap_err();
559        assert!(matches!(
560            error,
561            ProviderExecutionError::Cancelled {
562                stage: ExecutionStage::Collect
563            }
564        ));
565        assert_eq!(engine.adapter().cancel_calls.get(), 1);
566        assert_eq!(engine.adapter().collect_calls.get(), 0);
567    }
568
569    #[test]
570    fn cancellation_failure_is_not_hidden() {
571        let cancellation = ProcessCancellation::default();
572        let mut adapter = FakeAdapter::successful();
573        adapter.cancel_failure = true;
574        let engine = ProviderEngine::new(adapter);
575        let started = engine.start(&(), &cancellation).unwrap();
576        cancellation.cancel();
577        let error = engine.collect(started, &cancellation).unwrap_err();
578        assert!(matches!(
579            error,
580            ProviderExecutionError::Cancellation {
581                source: FakeFailure::Cancel
582            }
583        ));
584    }
585
586    #[test]
587    fn reconciliation_ambiguity_is_owned_by_the_engine() {
588        let mut adapter = FakeAdapter::successful();
589        adapter.unexpected_session = true;
590        let engine = ProviderEngine::new(adapter);
591        let error = engine
592            .execute(&(), &ProcessCancellation::default())
593            .unwrap_err();
594        assert!(matches!(
595            error,
596            ProviderExecutionError::Reconcile(ReconcileError::UnexpectedCollectedSession { .. })
597        ));
598    }
599
600    #[test]
601    fn partial_runs_preserve_receipts_and_present_payloads() {
602        let run = AdapterRun {
603            expected: vec![
604                ExpectedSession {
605                    session_id: "session-a".to_owned(),
606                    device_id: "device-a".to_owned(),
607                    status: "passed".to_owned(),
608                },
609                ExpectedSession {
610                    session_id: "session-b".to_owned(),
611                    device_id: "device-b".to_owned(),
612                    status: "passed".to_owned(),
613                },
614            ],
615            collected: vec![CollectedOutput {
616                session_id: "session-a".to_owned(),
617                reports: vec![11],
618                failure: None,
619            }],
620        }
621        .reconcile()
622        .unwrap();
623
624        assert_eq!(run.assessment().outcome(), MatrixOutcome::Partial);
625        assert_eq!(run.sessions().len(), 1);
626        assert_eq!(
627            run.assessment().sessions()[1].disposition,
628            SessionDisposition::Missing
629        );
630    }
631}