Skip to main content

termproof/
execution.rs

1//! ExecutionContext and ExecutionMode public traits (RUST-016).
2//!
3//! Execution modes receive an `ExecutionContext` that exposes only supported
4//! operations (session creation, step dispatch, assertion evaluation). They
5//! must not call private runner methods. This closes #78.
6//!
7//! Built-in modes are migrated to use the context.
8
9use std::collections::HashMap;
10use std::path::{Path, PathBuf};
11use std::time::Duration;
12
13use serde_json::Value;
14
15use crate::models::{AssertionResult, Recipe, StepResult};
16
17/// Result type for execution modes.
18pub type ExecutionResult = (
19    Vec<StepResult>,
20    Vec<AssertionResult>,
21    String,
22    Option<i32>,
23    String,
24);
25
26/// Errors from execution.
27#[derive(Debug, thiserror::Error)]
28pub enum ExecutionError {
29    /// Step failed to execute.
30    #[error("step error: {0}")]
31    Step(String),
32    /// Session backend error.
33    #[error("session error: {0}")]
34    Session(String),
35    /// Configuration or recipe error.
36    #[error("config error: {0}")]
37    Config(String),
38    /// Timeout.
39    #[error("timeout: {0}")]
40    Timeout(String),
41}
42
43/// The recipe's declared assertions plus the implicit `expect_exit_code` one.
44///
45/// Shared by the two `evaluate_assertions*` default bodies so the assertion
46/// list cannot drift between the step-aware and step-blind paths, and defined
47/// once in [`crate::assertions`] so it cannot drift from the list the harness
48/// measures either.
49fn assertions_with_implicit(recipe: &Recipe) -> Vec<Value> {
50    crate::assertions::evaluated_list(recipe)
51}
52
53/// Public context passed to execution modes.
54///
55/// This is the only interface execution modes may use. It hides runner
56/// internals and makes the boundary testable with an `InMemory` fake.
57pub trait ExecutionContext: Send {
58    /// Create a new terminal session via the configured backend.
59    fn create_session(
60        &mut self,
61        argv: Vec<String>,
62        cwd: Option<String>,
63        env: HashMap<String, String>,
64        cols: u16,
65        rows: u16,
66        cast_path: PathBuf,
67    ) -> Result<Box<dyn crate::terminal::Session>, ExecutionError>;
68
69    /// Execute a single step through the registered step registry.
70    fn run_step(
71        &mut self,
72        session: &mut dyn crate::terminal::Session,
73        step: &Value,
74        index: usize,
75    ) -> StepResult;
76
77    /// Evaluate a single assertion.
78    ///
79    /// The default body is [`crate::assertions::evaluate`], the one shared
80    /// implementation of the eight built-ins. Before it existed this was a
81    /// required method with no in-tree answer, so every execution mode was free
82    /// to answer differently and nothing measured whether they agreed. An
83    /// implementation that needs its own registry still overrides it.
84    fn evaluate_assertion(
85        &self,
86        recipe: &Recipe,
87        assertion: &Value,
88        screen: &str,
89        raw_output: &str,
90        exit_code: Option<i32>,
91    ) -> AssertionResult {
92        crate::assertions::evaluate(recipe, assertion, screen, raw_output, exit_code)
93    }
94
95    /// Evaluate a single assertion, with the screen captured after each step.
96    ///
97    /// This is the step-aware form of [`ExecutionContext::evaluate_assertion`].
98    /// `evaluate_assertion` only ever sees the final screen, so a state the
99    /// target passes through and then leaves — a palette that is opened and
100    /// then dismissed — is not expressible with it. `StepResult::screen`
101    /// already records the intermediate screens; this method is what carries
102    /// them to the assertion.
103    ///
104    /// The default body ignores `steps` and forwards to `evaluate_assertion`,
105    /// so an implementation written before this method existed keeps compiling
106    /// and keeps behaving identically. Overriding it is the opt-in.
107    ///
108    /// `steps` is `None` when the execution mode did not supply per-step
109    /// screens, which is not the same as an empty slice: an assertion that
110    /// needs them should report that they were unavailable rather than assume
111    /// no steps ran.
112    fn evaluate_assertion_with_steps(
113        &self,
114        recipe: &Recipe,
115        assertion: &Value,
116        screen: &str,
117        raw_output: &str,
118        exit_code: Option<i32>,
119        steps: Option<&[StepResult]>,
120    ) -> AssertionResult {
121        let _ = steps;
122        self.evaluate_assertion(recipe, assertion, screen, raw_output, exit_code)
123    }
124
125    /// Evaluate all assertions (including implicit `expect_exit_code`).
126    fn evaluate_assertions(
127        &self,
128        recipe: &Recipe,
129        screen: &str,
130        raw_output: &str,
131        exit_code: Option<i32>,
132    ) -> Vec<AssertionResult> {
133        assertions_with_implicit(recipe)
134            .iter()
135            .map(|a| self.evaluate_assertion(recipe, a, screen, raw_output, exit_code))
136            .collect()
137    }
138
139    /// Evaluate all assertions, with the screen captured after each step.
140    ///
141    /// The step-aware form of [`ExecutionContext::evaluate_assertions`]; it
142    /// assembles the same assertion list and routes each one through
143    /// [`ExecutionContext::evaluate_assertion_with_steps`]. An implementation
144    /// that overrides `evaluate_assertions` and wants the override to apply on
145    /// this path must override this method too.
146    fn evaluate_assertions_with_steps(
147        &self,
148        recipe: &Recipe,
149        screen: &str,
150        raw_output: &str,
151        exit_code: Option<i32>,
152        steps: Option<&[StepResult]>,
153    ) -> Vec<AssertionResult> {
154        assertions_with_implicit(recipe)
155            .iter()
156            .map(|a| {
157                self.evaluate_assertion_with_steps(recipe, a, screen, raw_output, exit_code, steps)
158            })
159            .collect()
160    }
161
162    /// Return the recipe's timeout as Duration.
163    ///
164    /// `timeout_seconds` comes out of the recipe file, so NaN, a negative and
165    /// a value past `Duration::from_secs_f64`'s ceiling are all reachable.
166    /// This uses the same clamping the step layer applies to its own
167    /// `timeout_seconds` (`spec/002-builtin-steps/spec.md` FR-006, FR-007):
168    /// non-positive and NaN are a deadline already past, and anything
169    /// enormous is clamped to the far future rather than panicking.
170    fn recipe_timeout(&self, recipe: &Recipe) -> Duration {
171        crate::steps::duration_from_secs(recipe.timeout_seconds)
172    }
173}
174
175/// Public execution-mode trait.
176///
177/// Implementations receive `&mut dyn ExecutionContext` and must not downcast
178/// to a concrete runner. This is enforced by taking a trait object.
179#[allow(clippy::type_complexity)]
180pub trait ExecutionMode: Send + Sync {
181    /// Mode name (e.g. "scripted_pty").
182    fn name(&self) -> &str;
183
184    /// Execute the recipe, returning steps, assertions, raw output, exit code, and screen.
185    fn execute(
186        &self,
187        ctx: &mut dyn ExecutionContext,
188        recipe: &Recipe,
189        run_dir: &Path,
190    ) -> Result<ExecutionResult, ExecutionError>;
191}
192
193/// PTY-based scripted execution mode.
194///
195/// Uses `ExecutionContext::create_session` and `run_step`; does not touch
196/// runner private methods.
197pub struct ScriptedPtyMode;
198
199impl ExecutionMode for ScriptedPtyMode {
200    fn name(&self) -> &str {
201        "scripted_pty"
202    }
203
204    fn execute(
205        &self,
206        ctx: &mut dyn ExecutionContext,
207        recipe: &Recipe,
208        run_dir: &Path,
209    ) -> Result<ExecutionResult, ExecutionError> {
210        let cast_path = run_dir.join("session.cast");
211        let mut session = ctx.create_session(
212            recipe.command.argv.clone(),
213            recipe.command.cwd.clone(),
214            recipe.command.env.clone(),
215            recipe.cols,
216            recipe.rows,
217            cast_path.clone(),
218        )?;
219        let mut steps = Vec::new();
220        for (i, step) in recipe.steps.iter().enumerate() {
221            let result = ctx.run_step(session.as_mut(), step, i + 1);
222            let passed = result.passed;
223            steps.push(result);
224            if !passed {
225                break;
226            }
227        }
228        let timeout = ctx.recipe_timeout(recipe);
229        if recipe.expect_exit_code.is_some() {
230            let _ = session.wait_for_exit(timeout);
231        } else {
232            let _ = session.wait_for_idle(
233                Duration::from_secs_f64(0.5),
234                timeout.min(Duration::from_secs(3)),
235            );
236        }
237        let raw_output = session.raw_output().to_string();
238        let exit_code = session.exit_code();
239        let screen = session.screen().to_string();
240        let assertions = ctx.evaluate_assertions_with_steps(
241            recipe,
242            &screen,
243            &raw_output,
244            exit_code,
245            Some(&steps),
246        );
247        let _ = session.close();
248        Ok((steps, assertions, raw_output, exit_code, screen))
249    }
250}
251
252/// Non-PTY process mode — same context-bound implementation.
253pub struct ScriptedProcessMode;
254
255impl ExecutionMode for ScriptedProcessMode {
256    fn name(&self) -> &str {
257        "scripted_process"
258    }
259
260    fn execute(
261        &self,
262        ctx: &mut dyn ExecutionContext,
263        recipe: &Recipe,
264        run_dir: &Path,
265    ) -> Result<ExecutionResult, ExecutionError> {
266        let cast_path = run_dir.join("session.cast");
267        let mut session = ctx.create_session(
268            recipe.command.argv.clone(),
269            recipe.command.cwd.clone(),
270            recipe.command.env.clone(),
271            recipe.cols,
272            recipe.rows,
273            cast_path,
274        )?;
275        let mut steps = Vec::new();
276        for (i, step) in recipe.steps.iter().enumerate() {
277            let result = ctx.run_step(session.as_mut(), step, i + 1);
278            let passed = result.passed;
279            steps.push(result);
280            if !passed {
281                break;
282            }
283        }
284        let _ = session.wait_for_exit(ctx.recipe_timeout(recipe));
285        let raw_output = session.raw_output().to_string();
286        let exit_code = session.exit_code();
287        let screen = session.screen().to_string();
288        let assertions = ctx.evaluate_assertions_with_steps(
289            recipe,
290            &screen,
291            &raw_output,
292            exit_code,
293            Some(&steps),
294        );
295        let _ = session.close();
296        Ok((steps, assertions, raw_output, exit_code, screen))
297    }
298}
299
300/// Agent-driven mode — delegates to an agent runner via the context.
301///
302/// In the Rust port the agent runner is invoked through a dedicated
303/// `AgentExecution` hook on the context; this simplified mode captures the
304/// PTY path for contract testing. Real subprocess work lives in
305/// `AgentDrivenRunner::run` (see `crate::agent`).
306pub struct AgentDrivenMode;
307
308impl ExecutionMode for AgentDrivenMode {
309    fn name(&self) -> &str {
310        "agent_driven"
311    }
312
313    fn execute(
314        &self,
315        ctx: &mut dyn ExecutionContext,
316        recipe: &Recipe,
317        run_dir: &Path,
318    ) -> Result<ExecutionResult, ExecutionError> {
319        let cast_path = run_dir.join("session.cast");
320        let mut session = ctx.create_session(
321            recipe.command.argv.clone(),
322            recipe.command.cwd.clone(),
323            recipe.command.env.clone(),
324            recipe.cols,
325            recipe.rows,
326            cast_path,
327        )?;
328        let prompt = crate::agent::build_agent_prompt(recipe);
329        let _ = std::fs::write(run_dir.join("agent_prompt.md"), &prompt);
330        let raw_output = format!(
331            "{{\"assertions\":{{}},\"transcript\":\"agent executed for recipe {}\"}}",
332            recipe.name
333        );
334        let parsed = crate::agent::parse_agent_output(&raw_output);
335        let _ = std::fs::write(run_dir.join("agent_transcript.md"), &parsed.transcript);
336        let screen = parsed.transcript.clone();
337        let checks = if recipe.checks.is_empty() {
338            vec!["Codex operator completed the verification".to_string()]
339        } else {
340            recipe.checks.clone()
341        };
342        let assertions = checks
343            .iter()
344            .map(|check| {
345                let passed = parsed.assertions.get(check).copied().unwrap_or(false);
346                AssertionResult {
347                    name: check.clone(),
348                    passed,
349                    detail: if passed {
350                        "agent reported pass".to_string()
351                    } else {
352                        "agent did not report pass".to_string()
353                    },
354                }
355            })
356            .collect();
357        let steps = vec![StepResult {
358            name: "codex-operator".to_string(),
359            passed: true,
360            detail: "agent mode via ExecutionContext".to_string(),
361            screen: screen.clone(),
362        }];
363        let _ = session.close();
364        Ok((steps, assertions, raw_output, Some(0), screen))
365    }
366}