Skip to main content

aion_worker/shell/
action.rs

1//! A declared command executed as an activity.
2//!
3//! [`ShellAction`] turns a command declared in an `.awl` worker block into a
4//! runnable activity, so an action can do real work without a hand-written
5//! Rust crate. The command is parsed once at construction and executed as a
6//! direct `execve` with no shell interposed — see [`super::template`] for why
7//! that is the security property rather than a convenience.
8
9use std::collections::BTreeMap;
10
11use serde::{Deserialize, Serialize};
12use tokio::process::Command;
13
14use super::exit::Ending;
15use super::failure::{spawn_failure, unreadable_ending_clause};
16use super::template::{CommandTemplate, SubstitutionError, TemplateError};
17use super::world::place_in_declared_world;
18use crate::activity::ActivityFailure;
19use crate::command_transcript::CommandTranscript;
20use crate::context::ActivityContext;
21use crate::process::{CancellableCommandOutput, run_cancellable_command};
22
23/// What a declared action produces when its command succeeds.
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
25pub struct ShellOutcome {
26    /// The command's exit status code.
27    pub exit_code: i32,
28    /// Everything the command wrote to standard output, trailing newline
29    /// trimmed so a one-line result is usable directly.
30    pub stdout: String,
31    /// Everything the command wrote to standard error, trailing newline
32    /// trimmed. Captured even on success, because a command that succeeds
33    /// while warning is a thing an operator needs to see.
34    pub stderr: String,
35}
36
37/// How a declared action treats a non-zero exit.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
39pub enum FailureMode {
40    /// A non-zero exit is retryable: the engine may run the command again
41    /// under the action's retry policy. This is the default because the
42    /// common causes of a failing command — a busy resource, an unreachable
43    /// host, a transient permission state — are the ones that succeed on a
44    /// second attempt.
45    #[default]
46    Retryable,
47    /// A non-zero exit is terminal: the command will never be retried. Use
48    /// when re-running the command could not possibly change the outcome, or
49    /// when running it twice would itself be harmful.
50    Terminal,
51}
52
53/// A declared command, ready to run as an activity.
54#[derive(Debug, Clone)]
55pub struct ShellAction {
56    template: CommandTemplate,
57    failure_mode: FailureMode,
58    environment: BTreeMap<String, String>,
59    working_directory: Option<std::path::PathBuf>,
60}
61
62impl ShellAction {
63    /// Parse `command` into a runnable action.
64    ///
65    /// # Errors
66    ///
67    /// Returns [`TemplateError`] when the command is empty, has an unterminated
68    /// quote or `${`, contains a `$` naming no parameter, or puts a parameter
69    /// in program position.
70    pub fn new(command: &str) -> Result<Self, TemplateError> {
71        Ok(Self {
72            template: CommandTemplate::parse(command)?,
73            failure_mode: FailureMode::default(),
74            environment: BTreeMap::new(),
75            working_directory: None,
76        })
77    }
78
79    /// Set how a non-zero exit is classified.
80    #[must_use]
81    pub const fn with_failure_mode(mut self, failure_mode: FailureMode) -> Self {
82        self.failure_mode = failure_mode;
83        self
84    }
85
86    /// Supply the environment the command runs with.
87    ///
88    /// Additive to nothing: the host's environment is cleared regardless, so
89    /// these entries plus [`super::world::INHERITED_VARIABLE`] are the
90    /// command's whole environment. An entry named `PATH` here replaces the
91    /// inherited one, because [`place_in_declared_world`] applies these after
92    /// the inherited value.
93    #[must_use]
94    pub fn with_environment(mut self, environment: BTreeMap<String, String>) -> Self {
95        self.environment = environment;
96        self
97    }
98
99    /// Run the command in `directory` instead of wherever the host happens to
100    /// be.
101    ///
102    /// Worth setting whenever a command names a relative path. Absent, the
103    /// command inherits the host process's working directory, which is a
104    /// property of how the host was launched and not something a package
105    /// author can see — so a relative path in a declared command means
106    /// something the author cannot predict.
107    #[must_use]
108    pub fn with_working_directory(mut self, directory: impl Into<std::path::PathBuf>) -> Self {
109        self.working_directory = Some(directory.into());
110        self
111    }
112
113    /// The parameter names this command references, sorted.
114    ///
115    /// A caller can compare these against an action's declared parameters to
116    /// catch a reference to a parameter that does not exist.
117    #[must_use]
118    pub fn referenced_parameters(&self) -> Vec<String> {
119        self.template.referenced_parameters()
120    }
121
122    /// Run the command with `arguments` bound to its parameter references.
123    ///
124    /// Cancellation is tree-wide: the command runs in its own process group,
125    /// and cancelling the activity terminates that whole group rather than
126    /// only the direct child, so a command that spawned children does not
127    /// leave them running.
128    ///
129    /// Output is streamed onto `context`'s transcript seam line by line as the
130    /// command writes it (see [`CommandTranscript`]), so an operator can read
131    /// what a command step is printing mid-run. The returned [`ShellOutcome`]
132    /// is unaffected: it still carries the command's complete stdout and stderr.
133    ///
134    /// # Errors
135    ///
136    /// Returns an [`ActivityFailure`] when a referenced parameter is missing or
137    /// unrepresentable (terminal — re-running cannot supply it), when the
138    /// command cannot be spawned or observed (terminal — the program is absent
139    /// or unrunnable), when the activity is already cancelled before the
140    /// command starts (terminal — nothing runs at all), when the command is
141    /// cancelled while running (terminal), when the host
142    /// reports an ending that is neither an exit nor a signal (terminal —
143    /// there is no honest status to report, and see
144    /// [`super::failure::ending_permits_retry`] for why an ending this host
145    /// cannot read is never retried at either executor), or when it exits
146    /// non-zero or is
147    /// ended by a signal (classified by the action's [`FailureMode`], and the
148    /// signal is named rather than rendered as an exit code alone).
149    pub async fn run(
150        &self,
151        arguments: &BTreeMap<String, serde_json::Value>,
152        context: &ActivityContext,
153    ) -> Result<ShellOutcome, ActivityFailure> {
154        // Read BEFORE anything is executed. A cancellation that has already
155        // landed must not start a program and then kill it: the point of
156        // cancelling is that the work does not happen. The residual window is
157        // stated rather than claimed away — a cancellation landing after this
158        // read still starts the program, and is answered by the termination
159        // ladder in the ordinary way.
160        if context.is_cancelled() {
161            return Err(ActivityFailure::terminal(
162                "this command was cancelled before it started, so it never ran",
163            ));
164        }
165        let argv = self
166            .template
167            .render(arguments)
168            .map_err(|error| substitution_failure(&error))?;
169        let (program, rest) = argv
170            .split_first()
171            // `CommandTemplate::parse` refuses an empty command, so a rendered
172            // argv always has a program. Handled rather than indexed so a
173            // future change to the parser cannot turn this into a panic.
174            .ok_or_else(|| ActivityFailure::terminal("the declared command rendered no program"))?;
175
176        let mut command = Command::new(program);
177        command.args(rest);
178        // Closed stdin, a cleared host environment, PATH, then this action's
179        // environment in order — the one world every declared command runs in,
180        // established by the one function that states it (see
181        // [`super::world`]).
182        place_in_declared_world(
183            &mut command,
184            self.environment
185                .iter()
186                .map(|(name, value)| (name.as_str(), value.as_str())),
187        );
188        if let Some(directory) = &self.working_directory {
189            command.current_dir(directory);
190        }
191
192        let transcript = CommandTranscript::new(context);
193        match run_cancellable_command(command, context.cancelled(), &transcript).await {
194            Ok(CancellableCommandOutput::Completed(output)) => {
195                let ending = Ending::of(output.status);
196                let stdout = trim_trailing_newline(&String::from_utf8_lossy(&output.stdout));
197                let stderr = trim_trailing_newline(&String::from_utf8_lossy(&output.stderr));
198                let Some(exit_code) = ending.reported_code() else {
199                    // Neither an exit code nor a signal: there is no number
200                    // this side could report without inventing one, and an
201                    // invented number is what an operator would act on.
202                    // TERMINAL, by the one classification both executors read
203                    // — `ending_permits_retry` is exactly this absence of a
204                    // reported code, and its doc carries the reasoning.
205                    return Err(ActivityFailure::terminal(format!(
206                        "the command `{program}` {ended}{unreadable}",
207                        ended = ending.described(),
208                        unreadable = unreadable_ending_clause(ending),
209                    )));
210                };
211                let outcome = ShellOutcome {
212                    exit_code,
213                    stdout,
214                    stderr,
215                };
216                if ending.succeeded() {
217                    Ok(outcome)
218                } else {
219                    Err(self.exit_failure(program, ending, &outcome))
220                }
221            }
222            Ok(CancellableCommandOutput::Cancelled) => Err(ActivityFailure::terminal(format!(
223                "the command `{program}` was cancelled: its process group was terminated and \
224                 proven gone, so nothing it started is still running"
225            ))),
226            Err(error) => Err(spawn_failure(program, None, &error)),
227        }
228    }
229
230    /// Build the failure for a command that ran and did not exit zero.
231    fn exit_failure(
232        &self,
233        program: &str,
234        ending: Ending,
235        outcome: &ShellOutcome,
236    ) -> ActivityFailure {
237        // The message carries HOW the command ended — an exit code, or the
238        // signal that ended it — and its own standard error, because a
239        // failing command's own words are the most useful thing an operator
240        // can be handed; without them the failure says only that something
241        // went wrong.
242        let ended = ending.described();
243        let message = if outcome.stderr.is_empty() {
244            format!("the command `{program}` {ended} and wrote nothing to standard error")
245        } else {
246            format!(
247                "the command `{program}` {ended} and wrote to standard error: {}",
248                outcome.stderr
249            )
250        };
251        match self.failure_mode {
252            FailureMode::Retryable => ActivityFailure::retryable(message),
253            FailureMode::Terminal => ActivityFailure::terminal(message),
254        }
255    }
256}
257
258/// Classify a substitution failure. All are terminal: the same call with the
259/// same arguments would fail identically however many times it is retried.
260fn substitution_failure(error: &SubstitutionError) -> ActivityFailure {
261    ActivityFailure::terminal(error.to_string())
262}
263
264/// Trim exactly one trailing newline, and its carriage return if present.
265///
266/// A command's output almost always ends in a newline that belongs to the
267/// terminal rather than to the value, so `echo hello` yields `hello` and not
268/// `hello\n`. Only one is removed: a command that deliberately emits blank
269/// trailing lines keeps them.
270pub(super) fn trim_trailing_newline(text: &str) -> String {
271    text.strip_suffix('\n')
272        .map_or(text, |trimmed| {
273            trimmed.strip_suffix('\r').unwrap_or(trimmed)
274        })
275        .to_owned()
276}
277
278#[cfg(test)]
279mod tests {
280    use super::{FailureMode, ShellAction, trim_trailing_newline};
281    use crate::activity::Classification;
282    use crate::context::ActivityContext;
283    use aion_core::{ActivityId, RunId, WorkflowId};
284    use std::collections::BTreeMap;
285
286    fn context() -> (ActivityContext, crate::context::ActivityCancellationHandle) {
287        ActivityContext::new(
288            WorkflowId::new_v4(),
289            RunId::new_v4(),
290            ActivityId::from_sequence_position(1),
291            1,
292        )
293    }
294
295    fn arguments(pairs: &[(&str, serde_json::Value)]) -> BTreeMap<String, serde_json::Value> {
296        pairs
297            .iter()
298            .map(|(name, value)| ((*name).to_owned(), value.clone()))
299            .collect()
300    }
301
302    /// What a test returns. Every fallible step is carried rather than
303    /// unwrapped, because the workspace denies panicking accessors in test
304    /// code as firmly as in library code.
305    type TestResult = Result<(), Box<dyn std::error::Error>>;
306
307    #[tokio::test]
308    async fn a_succeeding_command_returns_its_output() -> TestResult {
309        let action = ShellAction::new("echo hello")?;
310        let (context, _handle) = context();
311        let outcome = action.run(&BTreeMap::new(), &context).await?;
312        assert_eq!(outcome.exit_code, 0);
313        assert_eq!(outcome.stdout, "hello");
314        assert_eq!(outcome.stderr, "");
315        Ok(())
316    }
317
318    #[tokio::test]
319    async fn a_parameter_value_reaches_the_program_as_one_argument() -> TestResult {
320        let action = ShellAction::new("echo {{greeting}}")?;
321        let (context, _handle) = context();
322        let outcome = action
323            .run(
324                &arguments(&[("greeting", serde_json::json!("hello there world"))]),
325                &context,
326            )
327            .await?;
328        assert_eq!(outcome.stdout, "hello there world");
329        Ok(())
330    }
331
332    #[tokio::test]
333    async fn a_hostile_value_is_inert_because_no_shell_ever_sees_it() -> TestResult {
334        // If a shell were interposed, this would execute a second command.
335        // Because argv goes straight to execve, it is printed as text.
336        let action = ShellAction::new("echo {{value}}")?;
337        let (context, _handle) = context();
338        let outcome = action
339            .run(
340                &arguments(&[("value", serde_json::json!("hi; echo PWNED"))]),
341                &context,
342            )
343            .await?;
344        assert_eq!(outcome.stdout, "hi; echo PWNED");
345        assert!(
346            !outcome.stdout.contains("PWNED\n"),
347            "the injected command must never have run"
348        );
349        Ok(())
350    }
351
352    #[tokio::test]
353    async fn a_nonzero_exit_is_retryable_by_default_and_carries_stderr() -> TestResult {
354        let action = ShellAction::new("sh -c 'echo trouble >&2; exit 3'")?;
355        let (context, _handle) = context();
356        let Err(failure) = action.run(&BTreeMap::new(), &context).await else {
357            return Err("a non-zero exit must fail the activity".into());
358        };
359        assert_eq!(failure.classification(), &Classification::Retryable);
360        assert!(
361            failure.message().contains("trouble"),
362            "the failure must carry the command's own words: {}",
363            failure.message()
364        );
365        assert!(failure.message().contains('3'), "the exit code is reported");
366        Ok(())
367    }
368
369    #[tokio::test]
370    async fn a_nonzero_exit_is_terminal_when_the_action_says_so() -> TestResult {
371        let action = ShellAction::new("sh -c 'exit 1'")?.with_failure_mode(FailureMode::Terminal);
372        let (context, _handle) = context();
373        let Err(failure) = action.run(&BTreeMap::new(), &context).await else {
374            return Err("a non-zero exit must fail the activity".into());
375        };
376        assert_eq!(failure.classification(), &Classification::Terminal);
377        Ok(())
378    }
379
380    #[tokio::test]
381    async fn a_missing_parameter_fails_terminally_before_anything_runs() -> TestResult {
382        let action = ShellAction::new("echo {{absent}}")?;
383        let (context, _handle) = context();
384        let Err(failure) = action.run(&BTreeMap::new(), &context).await else {
385            return Err("a missing parameter must fail the activity".into());
386        };
387        assert_eq!(failure.classification(), &Classification::Terminal);
388        assert!(failure.message().contains("absent"));
389        Ok(())
390    }
391
392    #[tokio::test]
393    async fn an_absent_program_fails_terminally() -> TestResult {
394        let action = ShellAction::new("aion-no-such-program-exists-anywhere")?;
395        let (context, _handle) = context();
396        let Err(failure) = action.run(&BTreeMap::new(), &context).await else {
397            return Err("an absent program must fail the activity".into());
398        };
399        assert_eq!(failure.classification(), &Classification::Terminal);
400        Ok(())
401    }
402
403    #[tokio::test]
404    async fn cancellation_stops_the_command_and_fails_terminally() -> TestResult {
405        let action = ShellAction::new("sleep 30")?;
406        let (context, handle) = context();
407        let run = tokio::spawn(async move {
408            let (context, _keep) = (context, ());
409            action.run(&BTreeMap::new(), &context).await
410        });
411        // Give the command a moment to actually be running before cancelling,
412        // so this exercises termination rather than a pre-start short circuit.
413        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
414        handle.cancel();
415        let Err(failure) = run.await? else {
416            return Err("a cancelled command must fail the activity".into());
417        };
418        assert_eq!(failure.classification(), &Classification::Terminal);
419        assert!(failure.message().contains("cancelled"));
420        Ok(())
421    }
422
423    /// A cancellation that had ALREADY landed executes nothing: the program
424    /// never runs, rather than running and then being killed.
425    ///
426    /// Measured by a side effect on the filesystem rather than by the failure,
427    /// because a spawn-then-kill would report a cancellation too. The second
428    /// arm is the positive control — without it, a missing `touch` would make
429    /// the first arm pass vacuously.
430    #[tokio::test]
431    async fn a_cancellation_already_standing_runs_nothing_at_all() -> TestResult {
432        let directory = tempfile::tempdir()?;
433        let witness = directory.path().join("cancelled");
434        let control = directory.path().join("control");
435
436        // Both contexts are taken before either is used, so neither shadows
437        // the helper the other still needs.
438        let (cancelled_context, handle) = context();
439        let (control_context, _control_handle) = context();
440
441        let action = ShellAction::new(&format!("touch {}", witness.display()))?;
442        handle.cancel();
443        let Err(failure) = action.run(&BTreeMap::new(), &cancelled_context).await else {
444            return Err("a cancelled command must fail the activity".into());
445        };
446        assert_eq!(failure.classification(), &Classification::Terminal);
447        assert!(
448            failure.message().contains("cancelled"),
449            "a cancellation must be reported as one: {}",
450            failure.message()
451        );
452        assert!(
453            !witness.exists(),
454            "a cancellation that had already landed still executed the program"
455        );
456
457        let control_action = ShellAction::new(&format!("touch {}", control.display()))?;
458        control_action
459            .run(&BTreeMap::new(), &control_context)
460            .await?;
461        assert!(
462            control.exists(),
463            "the control failed: `touch` did not run even without a cancellation, so the first \
464             arm proves nothing"
465        );
466        Ok(())
467    }
468
469    #[tokio::test]
470    async fn the_hosts_environment_does_not_cross_into_a_declared_command() -> TestResult {
471        // A host process legitimately holds credentials. A deployed package
472        // must not be able to read them by declaring `run "env"`. This is the
473        // test that catches a regression to inherited environment.
474        //
475        // It reads a variable the host ALREADY has rather than setting one:
476        // `std::env::set_var` is `unsafe` (process-global, racy against every
477        // other thread) and the workspace denies unsafe outright. Any existing
478        // variable other than the deliberately-inherited PATH proves the same
479        // property.
480        let Some(present) = std::env::vars_os()
481            .filter_map(|(name, _)| name.into_string().ok())
482            .find(|name| name != "PATH" && !name.is_empty() && !name.contains('='))
483        else {
484            tracing::info!(
485                "skipping: the host has no environment variable besides PATH to prove \
486                 non-inheritance with"
487            );
488            return Ok(());
489        };
490        let action = ShellAction::new(&format!("sh -c 'echo \"[${{{present}:-absent}}]\"'"))?;
491        let (context, _handle) = context();
492        let outcome = action.run(&BTreeMap::new(), &context).await?;
493        assert_eq!(
494            outcome.stdout, "[absent]",
495            "the host's `{present}` leaked into a declared command"
496        );
497        Ok(())
498    }
499
500    #[tokio::test]
501    async fn an_operator_supplied_variable_does_reach_the_command() -> TestResult {
502        // The other half: clearing must not make the environment unusable.
503        let mut environment = BTreeMap::new();
504        environment.insert("DECLARED_GREETING".to_owned(), "supplied".to_owned());
505        let action = ShellAction::new("sh -c 'echo \"[$DECLARED_GREETING]\"'")?
506            .with_environment(environment);
507        let (context, _handle) = context();
508        let outcome = action.run(&BTreeMap::new(), &context).await?;
509        assert_eq!(outcome.stdout, "[supplied]");
510        Ok(())
511    }
512
513    #[tokio::test]
514    async fn a_command_runs_in_its_declared_working_directory() -> TestResult {
515        let action = ShellAction::new("pwd")?.with_working_directory("/");
516        let (context, _handle) = context();
517        let outcome = action.run(&BTreeMap::new(), &context).await?;
518        assert_eq!(outcome.stdout, "/");
519        Ok(())
520    }
521
522    #[tokio::test]
523    async fn a_command_reading_stdin_gets_end_of_file_rather_than_the_hosts() -> TestResult {
524        // Inherited stdin would hang the activity forever on an interactive
525        // boot. Closed stdin makes `cat` finish immediately with nothing.
526        let action = ShellAction::new("cat")?;
527        let (context, _handle) = context();
528        let outcome = action.run(&BTreeMap::new(), &context).await?;
529        assert_eq!(outcome.exit_code, 0);
530        assert_eq!(outcome.stdout, "");
531        Ok(())
532    }
533
534    #[tokio::test]
535    async fn awkward_values_survive_as_exactly_one_argument_each() -> TestResult {
536        // Cally's misparse question, not the injection one: a value that a
537        // careless argv build would split, truncate, or lose entirely. Each is
538        // printed back with a delimiter so a silent split is visible.
539        for (value, expected) in [
540            ("two\nlines", "[two\nlines]"),
541            ("", "[]"),
542            ("   leading and trailing   ", "[   leading and trailing   ]"),
543            ("tab\there", "[tab\there]"),
544            ("quote\"inside", "[quote\"inside]"),
545            ("single'inside", "[single'inside]"),
546            ("back\\slash", "[back\\slash]"),
547        ] {
548            let action = ShellAction::new("printf [%s] {{value}}")?;
549            let (context, _handle) = context();
550            let outcome = action
551                .run(&arguments(&[("value", serde_json::json!(value))]), &context)
552                .await?;
553            assert_eq!(
554                outcome.stdout, expected,
555                "value {value:?} did not arrive as exactly one argument"
556            );
557        }
558        Ok(())
559    }
560
561    #[tokio::test]
562    async fn a_value_that_looks_like_a_flag_is_still_passed_as_a_value() -> TestResult {
563        // A leading `-` is the misparse Apollo's law points at: the ARGV is
564        // correct (one element), but the PROGRAM may read it as an option.
565        // The executor cannot fix that — only the author can, by writing
566        // `--` in the declaration. Both halves are pinned here so the
567        // behaviour is recorded rather than discovered.
568        let exposed = ShellAction::new("printf %s {{value}}")?;
569        let (exposed_context, _exposed_handle) = context();
570        let outcome = exposed
571            .run(
572                &arguments(&[("value", serde_json::json!("-n"))]),
573                &exposed_context,
574            )
575            .await;
576        // `printf %s -n` — the program's own parse decides; the executor
577        // delivered exactly what was declared either way.
578        assert!(
579            outcome.is_ok(),
580            "the executor must deliver the value and let the program parse it"
581        );
582
583        // With `--` the author has told the program where options end, and the
584        // value is unambiguously a value.
585        let guarded = ShellAction::new("printf -- [%s] {{value}}")?;
586        let (guarded_context, _guarded_handle) = context();
587        let guarded_outcome = guarded
588            .run(
589                &arguments(&[("value", serde_json::json!("-n"))]),
590                &guarded_context,
591            )
592            .await?;
593        assert_eq!(guarded_outcome.stdout, "[-n]");
594        Ok(())
595    }
596
597    /// THE JOIN THIS BUILD MAKES, on the real declared-action path: a command
598    /// writing to BOTH streams has those lines on the activity's transcript seam
599    /// while it is still running — the same seam, envelope and stream key an
600    /// agent step publishes on.
601    ///
602    /// The run future is polled concurrently with the reads, so a capture that
603    /// only materialized at exit would resolve the run arm first and fail the
604    /// test by name. The command then sleeps until the test cancels it, which
605    /// also holds the pre-existing cancellation contract under streaming.
606    #[tokio::test]
607    async fn a_declared_command_streams_both_streams_before_it_exits() -> TestResult {
608        use aion_core::{RunId, WorkflowId};
609
610        let (sender, mut events) = tokio::sync::mpsc::unbounded_channel();
611        let (context, cancellation) = ActivityContext::with_transcript(
612            WorkflowId::new_v4(),
613            RunId::new_v4(),
614            ActivityId::from_sequence_position(9),
615            3,
616            sender,
617        );
618        let action = ShellAction::new("sh -c 'echo working; echo warning >&2; sleep 30'")?;
619        let no_arguments = BTreeMap::new();
620        let run = action.run(&no_arguments, &context);
621        tokio::pin!(run);
622
623        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
624        let mut observed: Vec<(String, String)> = Vec::new();
625        while !observed.iter().any(|(role, _)| role.contains("stdout"))
626            || !observed.iter().any(|(role, _)| role.contains("stderr"))
627        {
628            tokio::select! {
629                biased;
630                outcome = &mut run => {
631                    drop(outcome);
632                    return Err(format!(
633                        "the command completed before its output reached the transcript: \
634                         {observed:?}"
635                    )
636                    .into());
637                }
638                event = events.recv() => {
639                    let event = event.ok_or("the transcript seam closed mid-command")?;
640                    assert_eq!(event.attempt, 3, "the event carries the attempt it belongs to");
641                    assert_eq!(
642                        event.activity_id,
643                        ActivityId::from_sequence_position(9),
644                        "the event is keyed to the activity that is running"
645                    );
646                    let aion_core::ActivityEventKind::Message { text, .. } = event.kind else {
647                        return Err("an output line must be a Message".into());
648                    };
649                    observed.push((event.agent_role, text));
650                }
651                () = tokio::time::sleep_until(deadline) => {
652                    return Err(format!(
653                        "the running command's output never reached the transcript: {observed:?}"
654                    )
655                    .into());
656                }
657            }
658        }
659
660        assert!(
661            observed.contains(&("command stdout".to_owned(), "working".to_owned())),
662            "stdout must arrive labelled by its stream: {observed:?}"
663        );
664        assert!(
665            observed.contains(&("command stderr".to_owned(), "warning".to_owned())),
666            "stderr must arrive labelled by its stream: {observed:?}"
667        );
668
669        cancellation.cancel();
670        let Err(failure) = run.await else {
671            return Err("a cancelled command must fail the activity".into());
672        };
673        assert_eq!(failure.classification(), &Classification::Terminal);
674        Ok(())
675    }
676
677    #[test]
678    fn only_one_trailing_newline_is_trimmed() {
679        assert_eq!(trim_trailing_newline("hello\n"), "hello");
680        assert_eq!(trim_trailing_newline("hello\r\n"), "hello");
681        assert_eq!(trim_trailing_newline("hello\n\n"), "hello\n");
682        assert_eq!(trim_trailing_newline("hello"), "hello");
683        assert_eq!(trim_trailing_newline(""), "");
684    }
685}