Skip to main content

scientific_workflow/task/
unit.rs

1//! Application-owned scientific execution contract.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::sync::{Mutex, MutexGuard};
5
6use serde::de::DeserializeOwned;
7use serde_json::{Map, Value};
8use sha2::{Digest, Sha256};
9use thiserror::Error;
10
11use crate::observation::ObservationPlan;
12use crate::state::{StateSchemaProvider, SystemState, SystemStateSchema};
13
14use super::UnitResult;
15
16pub(crate) const SEED_DERIVATION_ALGORITHM: &str = "scientific-workflow.seed.v1";
17
18/// Immutable dependency results, execution facts, and optional seed derivation.
19///
20/// Workflow creates one context for each execution-unit initialization. A
21/// unit may inspect completed correlated dependencies. A stochastic unit
22/// requests only the named seeds it actually needs; Workflow derives them
23/// without counters or request-order dependence and records successful
24/// requests with the affected member's output metadata.
25pub struct InitializationContext {
26    master_seed: Option<u64>,
27    replicate_ordinal: u64,
28    task_identity: Box<str>,
29    execution_unit_key: Box<str>,
30    dependencies: super::dependencies::Dependencies,
31    requests: Mutex<BTreeMap<SeedRequest, u64>>,
32}
33
34impl InitializationContext {
35    #[cfg(test)]
36    pub(crate) fn new(
37        master_seed: Option<u64>,
38        replicate_ordinal: u64,
39        task_identity: &str,
40        execution_unit_key: &str,
41    ) -> Self {
42        Self::with_dependencies(
43            master_seed,
44            replicate_ordinal,
45            task_identity,
46            execution_unit_key,
47            Value::Array(Vec::new()),
48        )
49    }
50
51    pub(crate) fn with_dependencies(
52        master_seed: Option<u64>,
53        replicate_ordinal: u64,
54        task_identity: &str,
55        execution_unit_key: &str,
56        dependencies: Value,
57    ) -> Self {
58        Self {
59            master_seed,
60            replicate_ordinal,
61            task_identity: task_identity.into(),
62            execution_unit_key: execution_unit_key.into(),
63            dependencies: super::dependencies::Dependencies::from_json(dependencies)
64                .expect("Runtime supplies a validated dependency snapshot"),
65            requests: Mutex::new(BTreeMap::new()),
66        }
67    }
68
69    /// Returns completed summaries from the task's declared dependency phases.
70    pub const fn dependencies(&self) -> &super::dependencies::Dependencies {
71        &self.dependencies
72    }
73
74    /// Returns whether `wf_configs/study.json` supplied a master seed.
75    pub const fn has_master_seed(&self) -> bool {
76        self.master_seed.is_some()
77    }
78
79    /// Derives a seed shared by all members in this execution unit.
80    ///
81    /// `purpose` is a stable, nonempty semantic name such as `"pairing"`.
82    /// Repeating the same request returns the same seed and metadata entry.
83    pub fn shared_seed(&self, purpose: &str) -> Result<u64, SeedError> {
84        self.seed(SeedScope::Shared, purpose)
85    }
86
87    /// Derives a seed belonging to one member exposed by this execution unit.
88    ///
89    /// `member_identity` must exactly match that member's [`MemberView`] identity;
90    /// Workflow rejects an initialization that requested a seed for an unknown
91    /// member. `purpose` is a stable, nonempty semantic name such as
92    /// `"initialization"` or `"replacement"`.
93    pub fn member_seed(&self, member_identity: &str, purpose: &str) -> Result<u64, SeedError> {
94        validate_name(member_identity, "member identity")?;
95        self.seed(SeedScope::Member(member_identity.into()), purpose)
96    }
97
98    fn seed(&self, scope: SeedScope, purpose: &str) -> Result<u64, SeedError> {
99        validate_name(purpose, "purpose")?;
100        let master_seed = self.master_seed.ok_or(SeedError::MissingMasterSeed)?;
101        let request = SeedRequest {
102            scope,
103            purpose: purpose.into(),
104        };
105        if let Some(seed) = self.request_ledger().get(&request) {
106            return Ok(*seed);
107        }
108
109        let seed = derive_seed(
110            master_seed,
111            self.replicate_ordinal,
112            &self.task_identity,
113            &self.execution_unit_key,
114            &request.scope,
115            &request.purpose,
116        );
117        self.request_ledger().insert(request, seed);
118        Ok(seed)
119    }
120
121    pub(crate) fn validate_member_identities<'a>(
122        &self,
123        identities: impl IntoIterator<Item = &'a str>,
124    ) -> Result<(), SeedError> {
125        let identities = identities.into_iter().collect::<BTreeSet<_>>();
126        for request in self.request_ledger().keys() {
127            if let SeedScope::Member(identity) = &request.scope
128                && !identities.contains(identity.as_ref())
129            {
130                return Err(SeedError::UnknownMemberIdentity {
131                    identity: identity.to_string(),
132                });
133            }
134        }
135        Ok(())
136    }
137
138    pub(crate) fn metadata_for_member(&self, member_identity: &str) -> Option<Value> {
139        let requests = self
140            .request_ledger()
141            .iter()
142            .filter_map(|(request, seed)| match &request.scope {
143                SeedScope::Shared => Some(Value::Object(Map::from_iter([
144                    ("scope".to_owned(), "shared".into()),
145                    ("purpose".to_owned(), request.purpose.to_string().into()),
146                    ("seed".to_owned(), (*seed).into()),
147                ]))),
148                SeedScope::Member(identity) if identity.as_ref() == member_identity => {
149                    Some(Value::Object(Map::from_iter([
150                        ("scope".to_owned(), "member".into()),
151                        ("member_identity".to_owned(), identity.to_string().into()),
152                        ("purpose".to_owned(), request.purpose.to_string().into()),
153                        ("seed".to_owned(), (*seed).into()),
154                    ])))
155                }
156                SeedScope::Member(_) => None,
157            })
158            .collect::<Vec<_>>();
159        (!requests.is_empty()).then(|| {
160            Value::Object(Map::from_iter([
161                ("algorithm".to_owned(), SEED_DERIVATION_ALGORITHM.into()),
162                (
163                    "master_seed".to_owned(),
164                    self.master_seed
165                        .expect("a recorded request has a seed")
166                        .into(),
167                ),
168                ("requests".to_owned(), requests.into()),
169            ]))
170        })
171    }
172
173    fn request_ledger(&self) -> MutexGuard<'_, BTreeMap<SeedRequest, u64>> {
174        self.requests
175            .lock()
176            .unwrap_or_else(|error| error.into_inner())
177    }
178}
179
180pub(crate) fn derive_program_seed(
181    master_seed: u64,
182    replicate_ordinal: u64,
183    task_identity: &str,
184    program_kind: &str,
185    purpose: &str,
186) -> u64 {
187    derive_seed(
188        master_seed,
189        replicate_ordinal,
190        task_identity,
191        program_kind,
192        &SeedScope::Shared,
193        purpose,
194    )
195}
196
197fn derive_seed(
198    master_seed: u64,
199    replicate_ordinal: u64,
200    task_identity: &str,
201    workload_key: &str,
202    scope: &SeedScope,
203    purpose: &str,
204) -> u64 {
205    let mut digest = Sha256::new();
206    hash_part(&mut digest, SEED_DERIVATION_ALGORITHM.as_bytes());
207    digest.update(master_seed.to_le_bytes());
208    digest.update(replicate_ordinal.to_le_bytes());
209    hash_part(&mut digest, task_identity.as_bytes());
210    hash_part(&mut digest, workload_key.as_bytes());
211    match scope {
212        SeedScope::Shared => digest.update([0]),
213        SeedScope::Member(identity) => {
214            digest.update([1]);
215            hash_part(&mut digest, identity.as_bytes());
216        }
217    }
218    hash_part(&mut digest, purpose.as_bytes());
219    u64::from_le_bytes(digest.finalize()[..8].try_into().expect("SHA-256 prefix"))
220}
221
222fn hash_part(digest: &mut Sha256, bytes: &[u8]) {
223    digest.update((bytes.len() as u64).to_le_bytes());
224    digest.update(bytes);
225}
226
227fn validate_name(value: &str, field: &'static str) -> Result<(), SeedError> {
228    if value.is_empty() || value.trim() != value {
229        return Err(SeedError::InvalidName {
230            field,
231            value: value.to_owned(),
232        });
233    }
234    Ok(())
235}
236
237#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
238struct SeedRequest {
239    scope: SeedScope,
240    purpose: Box<str>,
241}
242
243#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
244enum SeedScope {
245    Shared,
246    Member(Box<str>),
247}
248
249/// Failure to make or validate a deterministic initialization seed request.
250#[derive(Debug, Error)]
251#[non_exhaustive]
252pub enum SeedError {
253    /// The execution unit requested a seed but the study has none.
254    #[error("this execution unit requires `seed` in `wf_configs/study.json`")]
255    MissingMasterSeed,
256    /// A stable seed name was empty or had surrounding whitespace.
257    #[error("invalid seed {field} `{value}`: it must be nonempty with no surrounding whitespace")]
258    InvalidName {
259        /// The invalid semantic field.
260        field: &'static str,
261        /// The rejected value.
262        value: String,
263    },
264    /// A member seed was requested for an identity the unit did not expose.
265    #[error("seed requested for unknown member identity `{identity}`")]
266    UnknownMemberIdentity {
267        /// The requested identity absent from the execution unit.
268        identity: String,
269    },
270}
271
272/// The optional completion declaration for one execution-unit member.
273///
274/// This borrowed value distinguishes an incomplete member from a completed
275/// member with or without a structured reason. The execution unit retains the
276/// reason object; Workflow copies it only when the member first completes.
277#[derive(Clone, Copy, Debug)]
278pub struct MemberCompletion<'a> {
279    reason: Option<&'a Map<String, Value>>,
280}
281
282impl<'a> MemberCompletion<'a> {
283    /// Declares completion without a structured reason.
284    pub const fn without_reason() -> Self {
285        Self { reason: None }
286    }
287
288    /// Declares completion with a borrowed structured JSON reason.
289    pub const fn with_reason(reason: &'a Map<String, Value>) -> Self {
290        Self {
291            reason: Some(reason),
292        }
293    }
294
295    /// Returns the structured reason, when the execution unit supplied one.
296    pub const fn reason(self) -> Option<&'a Map<String, Value>> {
297        self.reason
298    }
299}
300
301/// A borrowed view of one independently stateful member inside an execution unit.
302///
303/// The identity, state owner, and schema allocation must remain stable at the
304/// same member index for the complete execution. `completion` and
305/// `target_iteration` are declarations inspected by Workflow; constructing a
306/// view has no side effects.
307#[derive(Clone, Copy, Debug)]
308pub struct MemberView<'a> {
309    identity: &'a str,
310    state: &'a SystemState,
311    completion: Option<MemberCompletion<'a>>,
312    target_iteration: Option<u64>,
313}
314
315impl<'a> MemberView<'a> {
316    /// Describes one member owned by an [`ExecutionUnit`].
317    ///
318    /// `identity` must be nonempty, contain no surrounding whitespace, and be
319    /// unique within the unit. `state` must remain at the same address and use
320    /// the schema supplied to [`ExecutionUnit::initialize`]. A target, when
321    /// present, must not precede the state's current iteration.
322    pub const fn new(
323        identity: &'a str,
324        state: &'a SystemState,
325        completion: Option<MemberCompletion<'a>>,
326        target_iteration: Option<u64>,
327    ) -> Self {
328        Self {
329            identity,
330            state,
331            completion,
332            target_iteration,
333        }
334    }
335
336    /// Returns the stable identity of this member within its execution unit.
337    pub const fn identity(self) -> &'a str {
338        self.identity
339    }
340
341    /// Borrows this member's directly owned canonical state.
342    pub const fn state(self) -> &'a SystemState {
343        self.state
344    }
345
346    /// Returns completion details, or `None` while the member is incomplete.
347    pub const fn completion(self) -> Option<MemberCompletion<'a>> {
348        self.completion
349    }
350
351    /// Returns this member's optional expected final iteration.
352    pub const fn target_iteration(self) -> Option<u64> {
353        self.target_iteration
354    }
355}
356
357/// One schedulable scientific execution containing one or more members.
358///
359/// Workflow manages every implementation through the same lifecycle and does
360/// not distinguish a standalone unit from a coordinated ensemble. A normal
361/// unit returns one [`MemberView`]; an ensemble returns one view per member and
362/// keeps all internal parallelism, shared inputs, and synchronization private.
363/// Each exposed member owns a distinct [`SystemState`]; Workflow rejects
364/// duplicate state addresses before beginning member recordings.
365///
366/// Member count, index order, identities, state owners, and schema allocations
367/// must remain stable after initialization. One successful [`Self::step`] must
368/// strictly advance at least one incomplete member and must never advance a
369/// member that was already complete. Other incomplete members may wait during a
370/// coordinated step, which permits synchronized ensembles and restored members
371/// at different iterations.
372pub trait ExecutionUnit: Send + Sized + 'static {
373    /// One complete set of constants supplied by Config.
374    type Constants: DeserializeOwned + 'static;
375
376    /// Declares whether changing Rayon worker counts between coordinated steps
377    /// preserves this unit's scientific result.
378    ///
379    /// Automatic compute allocation requires an explicit `true`; the default
380    /// keeps fixed-width numerical units out of dynamically rebalanced runs.
381    const THREAD_COUNT_INVARIANT: bool = false;
382
383    /// Returns the standard static schema supplied for this unit, if any.
384    ///
385    /// Study uses this provider only when the task omits an explicit project
386    /// `state`. The default requires callers to select a project schema. An
387    /// upstream provider owns the JSON document; this unit merely declares
388    /// that it can receive states using that standard layout.
389    fn standard_state_schema() -> Option<StateSchemaProvider> {
390        None
391    }
392
393    /// Validates unit-owned configuration and defines per-member observations.
394    ///
395    /// All members of one execution unit use the state schema selected by the
396    /// task and this common observation plan. The default records every field
397    /// at every iteration without additional validation. An override owns all
398    /// domain validation that can be performed before initialization; Study
399    /// trusts a successful result. This preflight operation must not create
400    /// output, initialize the unit, or mutate external state.
401    fn preflight(
402        _constants: &Self::Constants,
403        _schema: &SystemStateSchema,
404    ) -> UnitResult<ObservationPlan> {
405        Ok(ObservationPlan::all_fields())
406    }
407
408    /// Builds a fully initialized standalone unit or ensemble.
409    ///
410    /// Every state subsequently exposed through [`Self::member`] must have been
411    /// created from this exact schema allocation.
412    fn initialize(
413        constants: Self::Constants,
414        schema: &SystemStateSchema,
415        context: &InitializationContext,
416    ) -> UnitResult<Self>;
417
418    /// Returns the stable positive number of independently stateful members.
419    fn member_count(&self) -> usize;
420
421    /// Borrows one member by stable zero-based index.
422    ///
423    /// The method must be side-effect free. Workflow calls it repeatedly before
424    /// and after coordinated steps. An index below [`Self::member_count`] must
425    /// always return `Some`; all other indices must return `None`.
426    fn member(&self, index: usize) -> Option<MemberView<'_>>;
427
428    /// Performs one coordinated scientific transition.
429    ///
430    /// A standalone unit advances itself. An ensemble may advance members in
431    /// parallel or share generated inputs, but it must return only after the
432    /// complete logical step is visible through [`Self::member`].
433    fn step(&mut self) -> UnitResult;
434}
435
436#[cfg(test)]
437mod tests {
438    use super::*;
439
440    #[test]
441    fn named_seed_derivation_is_stable_and_request_order_independent() {
442        let first = InitializationContext::new(Some(99), 3, "phase/task", "ensemble");
443        let first_member = first.member_seed("alpha", "initialization").unwrap();
444        let first_shared = first.shared_seed("pairing").unwrap();
445        assert_eq!(first_member, 16_741_472_295_366_384_357);
446        assert_eq!(first_shared, 1_632_703_961_247_452_931);
447
448        let second = InitializationContext::new(Some(99), 3, "phase/task", "ensemble");
449        assert_eq!(second.shared_seed("pairing").unwrap(), first_shared);
450        assert_eq!(
451            second.member_seed("alpha", "initialization").unwrap(),
452            first_member
453        );
454        assert_ne!(
455            second.member_seed("beta", "initialization").unwrap(),
456            first_member
457        );
458
459        let next_replicate = InitializationContext::new(Some(99), 4, "phase/task", "ensemble");
460        assert_ne!(next_replicate.shared_seed("pairing").unwrap(), first_shared);
461    }
462
463    #[test]
464    fn deterministic_units_need_no_master_seed_but_requests_do() {
465        let context = InitializationContext::new(None, 0, "task", "unit");
466        assert!(!context.has_master_seed());
467        assert!(matches!(
468            context.shared_seed("pairing"),
469            Err(SeedError::MissingMasterSeed)
470        ));
471        assert!(context.metadata_for_member("member").is_none());
472    }
473
474    #[test]
475    fn program_seed_derivation_uses_task_kind_purpose_and_replicate() {
476        let seed = derive_program_seed(99, 3, "phase/task", "program", "preparation");
477        assert_eq!(
478            derive_program_seed(99, 3, "phase/task", "program", "preparation"),
479            seed
480        );
481        assert_ne!(
482            derive_program_seed(99, 4, "phase/task", "program", "preparation"),
483            seed
484        );
485        assert_ne!(
486            derive_program_seed(99, 3, "phase/task", "python", "preparation"),
487            seed
488        );
489        assert_ne!(
490            derive_program_seed(99, 3, "phase/task", "program", "analysis"),
491            seed
492        );
493    }
494}