speccoder_runtime/substrate/
run_config.rs

1//! `[budget]` and `[commands]` in `speccoder.toml` (policy).
2
3use serde::Deserialize;
4
5#[derive(Debug, Default, Deserialize)]
6#[serde(deny_unknown_fields)]
7struct RawBudget {
8    max_turns: Option<u32>,
9    max_tool_calls: Option<u32>,
10    max_attempts: Option<u32>,
11    best_of: Option<u32>,
12}
13
14#[derive(Debug, Default, Deserialize)]
15#[serde(deny_unknown_fields)]
16struct RawAcceptance {
17    generate: Option<bool>,
18    held_out: Option<bool>,
19    strength: Option<bool>,
20    min_strength: Option<f64>,
21    near_miss: Option<u32>,
22    permitted: Option<u32>,
23    strength_timeout_secs: Option<u64>,
24}
25
26/// How the acceptance set's strength is measured: `[acceptance]`
27/// `strength`, `min_strength`, `near_miss`, `permitted`,
28/// `strength_timeout_secs`.
29#[derive(Debug, Clone, PartialEq)]
30pub struct StrengthConfig {
31    /// Measure after every `verified` verdict (default on).
32    pub enabled: bool,
33    /// The gate; `None` (default): strength never changes a verdict.
34    pub min: Option<f64>,
35    /// Wrong variants asked for (0–3, default 2).
36    pub near_miss: u32,
37    /// Spec-permitted alternatives asked for (0–2, default 1; 0: no
38    /// over-strict check).
39    pub permitted: u32,
40    /// The whole stage's budget.
41    pub timeout: std::time::Duration,
42}
43
44// `min` is validated to 0.0–1.0, so it is never NaN.
45impl Eq for StrengthConfig {}
46
47impl Default for StrengthConfig {
48    fn default() -> Self {
49        Self {
50            enabled: true,
51            min: None,
52            near_miss: 2,
53            permitted: 1,
54            timeout: std::time::Duration::from_secs(900),
55        }
56    }
57}
58
59#[derive(Debug, Default, Deserialize)]
60#[serde(deny_unknown_fields)]
61struct RawFormat {
62    argv: Vec<String>,
63}
64
65#[derive(Debug, Default, Deserialize)]
66#[serde(deny_unknown_fields)]
67struct RawTests {
68    frozen: Option<String>,
69}
70
71#[derive(Debug, Default, Deserialize)]
72#[serde(deny_unknown_fields)]
73struct RawMonitor {
74    enabled: Option<bool>,
75}
76
77#[derive(Debug, Default, Deserialize)]
78#[serde(deny_unknown_fields)]
79struct RawContext {
80    instructions: Option<Vec<String>>,
81}
82
83#[derive(Debug, Default, Deserialize)]
84#[serde(deny_unknown_fields)]
85struct RawConverge {
86    max_rounds: Option<u32>,
87}
88
89#[derive(Debug, Default, Deserialize)]
90#[serde(deny_unknown_fields)]
91struct RawReview {
92    required: Option<bool>,
93}
94
95#[derive(Debug, Default, Deserialize)]
96#[serde(deny_unknown_fields)]
97struct RawAttest {
98    enabled: Option<bool>,
99    signer: Option<String>,
100    sign_at: Option<String>,
101    sigstore: Option<RawSigstore>,
102}
103
104#[derive(Debug, Default, Deserialize)]
105#[serde(deny_unknown_fields)]
106struct RawSigstore {
107    identity: Option<String>,
108    issuer: Option<String>,
109}
110
111/// Who signs a task's attestation (`[attest] signer`).
112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
113pub enum Signer {
114    /// The person's git signing key (SSH or OpenPGP), as git would use it.
115    Git,
116    /// Sigstore keyless, through `cosign`.
117    Sigstore,
118    /// Nobody: statements are stored unsigned.
119    None,
120}
121
122impl Signer {
123    /// The configured word.
124    #[must_use]
125    pub const fn word(self) -> &'static str {
126        match self {
127            Self::Git => "git",
128            Self::Sigstore => "sigstore",
129            Self::None => "none",
130        }
131    }
132}
133
134/// `[attest]`: signed evidence per ticked task.
135#[derive(Debug, Clone, PartialEq, Eq)]
136pub struct AttestConfig {
137    /// Write attestations on ticks.
138    pub enabled: bool,
139    /// Who signs.
140    pub signer: Signer,
141    /// Sign at the tick, or later with `speccoder attest sign` (touch keys).
142    pub sign_later: bool,
143    /// The Sigstore certificate identity the project says signs: shown to
144    /// reviewers (`pr`) to confirm, never trusted by `attest verify`.
145    pub sigstore_identity: Option<String>,
146    /// That identity's OIDC issuer, likewise.
147    pub sigstore_issuer: Option<String>,
148}
149
150impl Default for AttestConfig {
151    fn default() -> Self {
152        Self {
153            enabled: true,
154            signer: Signer::Git,
155            sign_later: false,
156            sigstore_identity: None,
157            sigstore_issuer: None,
158        }
159    }
160}
161
162#[derive(Debug, Default, Deserialize)]
163#[serde(deny_unknown_fields)]
164struct RawApprovals {
165    wait: Option<String>,
166}
167
168#[derive(Debug, Default, Deserialize)]
169#[serde(deny_unknown_fields)]
170struct RawGit {
171    commit: Option<String>,
172}
173
174#[derive(Debug, Default, Deserialize)]
175#[serde(deny_unknown_fields)]
176struct RawCommands {
177    #[serde(default)]
178    allow: Vec<Vec<String>>,
179}
180
181/// Limits and command allowances for an attempt.
182#[derive(Debug, Clone, PartialEq, Eq)]
183pub struct RunConfig {
184    /// Model turns per attempt.
185    pub max_turns: u32,
186    /// Tool calls per attempt.
187    pub max_tool_calls: u32,
188    /// Attempts per task run; failed verdicts are retried up to this.
189    pub max_attempts: u32,
190    /// Independent samples per task, the best by evidence kept.
191    pub best_of: u32,
192    /// Extra allowed argv prefixes for `run_command`.
193    pub allow: Vec<Vec<String>>,
194    /// Commit each passed task on the speccoder branch (`[git] commit = "per-task"`).
195    pub commit_per_task: bool,
196    /// Derive acceptance tests from the spec before attempts (`[acceptance] generate`).
197    pub acceptance: bool,
198    /// Also author held-out acceptance tests the implementer never sees
199    /// (`[acceptance] held_out`, default on; nothing without `generate`).
200    pub held_out: bool,
201    /// Acceptance strength.
202    pub strength: StrengthConfig,
203    /// Convergence phases a feature may get (`[converge] max_rounds`).
204    pub max_converge_rounds: u32,
205    /// Ask the gaming monitor about every passing task (`[monitor] enabled`).
206    pub monitor: bool,
207    /// A run waits until every requirement is approved on its current
208    /// wording (`[review] required`).
209    pub review_required: bool,
210    /// Test files that existed before an attempt may gain lines without
211    /// approval (`[tests] frozen = "additions"`; default `"ask"`).
212    pub frozen_additions: bool,
213    /// The formatter speccoder runs after each attempt, before verifying
214    /// (`[format] argv`): formatting is not the model's job to remember.
215    pub format: Option<Vec<String>>,
216    /// How long a call waits for a person before it is denied as expired
217    /// (`[approvals] wait`, default 7 days).
218    pub approval_wait: std::time::Duration,
219    /// `[attest]`.
220    pub attest: AttestConfig,
221    /// `[context] instructions`: the file names read per directory as the
222    /// repository's instructions for agents; empty: none.
223    pub instructions: Vec<String>,
224}
225
226impl Default for RunConfig {
227    fn default() -> Self {
228        Self {
229            max_turns: 30,
230            max_tool_calls: 100,
231            max_attempts: 2,
232            best_of: 1,
233            allow: Vec::new(),
234            commit_per_task: true,
235            acceptance: true,
236            held_out: true,
237            strength: StrengthConfig::default(),
238            max_converge_rounds: 3,
239            monitor: false,
240            review_required: false,
241            frozen_additions: false,
242            format: None,
243            approval_wait: std::time::Duration::from_secs(7 * 24 * 3600),
244            attest: AttestConfig::default(),
245            instructions: vec!["AGENTS.md".to_owned()],
246        }
247    }
248}
249
250/// `30m`, `12h` or `7d`; never zero.
251fn wait(text: &str) -> Result<std::time::Duration, String> {
252    let text = text.trim();
253    // The unit is the last character, which may be more than a byte (`7д`).
254    let (n, unit) = text.split_at(text.char_indices().last().map_or(0, |(i, _)| i));
255    let secs = match unit {
256        "m" => 60,
257        "h" => 3600,
258        "d" => 86_400,
259        _ => 0,
260    };
261    match n.parse::<u64>() {
262        Ok(n) if n > 0 && secs > 0 => Ok(std::time::Duration::from_secs(n.saturating_mul(secs))),
263        _ => Err(format!(
264            "[approvals]: wait = \"{text}\" (a positive number of m, h or d, e.g. \"7d\")"
265        )),
266    }
267}
268
269/// Reads `[budget]` and `[commands]`; absent sections give defaults.
270pub fn parse(text: &str) -> Result<RunConfig, String> {
271    let table: toml::Table = toml::from_str(text).map_err(|e| e.to_string())?;
272    let budget: RawBudget = match table.get("budget") {
273        Some(v) => v.clone().try_into().map_err(|e| format!("[budget]: {e}"))?,
274        None => RawBudget::default(),
275    };
276    let commands: RawCommands = match table.get("commands") {
277        Some(v) => v
278            .clone()
279            .try_into()
280            .map_err(|e| format!("[commands]: {e}"))?,
281        None => RawCommands::default(),
282    };
283    let git: RawGit = match table.get("git") {
284        Some(v) => v.clone().try_into().map_err(|e| format!("[git]: {e}"))?,
285        None => RawGit::default(),
286    };
287    let commit_per_task = match git.commit.as_deref() {
288        None | Some("per-task") => true,
289        Some("none") => false,
290        Some(other) => return Err(format!("[git]: commit = \"{other}\" (per-task or none)")),
291    };
292    let acceptance: RawAcceptance = match table.get("acceptance") {
293        Some(v) => v
294            .clone()
295            .try_into()
296            .map_err(|e| format!("[acceptance]: {e}"))?,
297        None => RawAcceptance::default(),
298    };
299    let converge: RawConverge = match table.get("converge") {
300        Some(v) => v
301            .clone()
302            .try_into()
303            .map_err(|e| format!("[converge]: {e}"))?,
304        None => RawConverge::default(),
305    };
306    let review: RawReview = match table.get("review") {
307        Some(v) => v.clone().try_into().map_err(|e| format!("[review]: {e}"))?,
308        None => RawReview::default(),
309    };
310    let monitor: RawMonitor = match table.get("monitor") {
311        Some(v) => v
312            .clone()
313            .try_into()
314            .map_err(|e| format!("[monitor]: {e}"))?,
315        None => RawMonitor::default(),
316    };
317    let tests: RawTests = match table.get("tests") {
318        Some(v) => v.clone().try_into().map_err(|e| format!("[tests]: {e}"))?,
319        None => RawTests::default(),
320    };
321    let frozen_additions = match tests.frozen.as_deref() {
322        None | Some("ask") => false,
323        Some("additions") => true,
324        Some(other) => {
325            return Err(format!("[tests]: frozen = \"{other}\" (ask or additions)"));
326        }
327    };
328    let format: Option<RawFormat> = match table.get("format") {
329        Some(v) => Some(v.clone().try_into().map_err(|e| format!("[format]: {e}"))?),
330        None => None,
331    };
332    if format
333        .as_ref()
334        .is_some_and(|f| f.argv.first().is_none_or(String::is_empty))
335    {
336        return Err("[format]: `argv` must name a program".to_owned());
337    }
338    let approvals: RawApprovals = match table.get("approvals") {
339        Some(v) => v
340            .clone()
341            .try_into()
342            .map_err(|e| format!("[approvals]: {e}"))?,
343        None => RawApprovals::default(),
344    };
345    let raw_attest: RawAttest = match table.get("attest") {
346        Some(v) => v.clone().try_into().map_err(|e| format!("[attest]: {e}"))?,
347        None => RawAttest::default(),
348    };
349    let attest = AttestConfig {
350        enabled: raw_attest.enabled.unwrap_or(true),
351        signer: match raw_attest.signer.as_deref() {
352            None | Some("git") => Signer::Git,
353            Some("sigstore") => Signer::Sigstore,
354            Some("none") => Signer::None,
355            Some(other) => {
356                return Err(format!(
357                    "[attest]: signer = \"{other}\" (git, sigstore or none)"
358                ));
359            }
360        },
361        sign_later: match raw_attest.sign_at.as_deref() {
362            None | Some("tick") => false,
363            Some("later") => true,
364            Some(other) => {
365                return Err(format!("[attest]: sign_at = \"{other}\" (tick or later)"));
366            }
367        },
368        sigstore_identity: raw_attest
369            .sigstore
370            .as_ref()
371            .and_then(|s| s.identity.clone()),
372        sigstore_issuer: raw_attest.sigstore.and_then(|s| s.issuer),
373    };
374    let context: RawContext = match table.get("context") {
375        Some(v) => v
376            .clone()
377            .try_into()
378            .map_err(|e| format!("[context]: {e}"))?,
379        None => RawContext::default(),
380    };
381    if let Some(bad) = context.instructions.iter().flatten().find(|n| {
382        n.is_empty() || n.contains('/') || n.contains('\\') || matches!(n.as_str(), "." | "..")
383    }) {
384        return Err(format!(
385            "[context]: instructions = \"{bad}\": a file name such as AGENTS.md, without a directory"
386        ));
387    }
388    let d = RunConfig::default();
389    let strength = {
390        let ds = StrengthConfig::default();
391        if let Some(m) = acceptance.min_strength
392            && !(0.0..=1.0).contains(&m)
393        {
394            return Err(format!(
395                "[acceptance]: min_strength = {m} (a share from 0.0 to 1.0)"
396            ));
397        }
398        let near_miss = acceptance.near_miss.unwrap_or(ds.near_miss);
399        if near_miss > 3 {
400            return Err(format!("[acceptance]: near_miss = {near_miss} (0 to 3)"));
401        }
402        let permitted = acceptance.permitted.unwrap_or(ds.permitted);
403        if permitted > 2 {
404            return Err(format!("[acceptance]: permitted = {permitted} (0 to 2)"));
405        }
406        StrengthConfig {
407            enabled: acceptance.strength.unwrap_or(ds.enabled),
408            min: acceptance.min_strength,
409            near_miss,
410            permitted,
411            timeout: acceptance
412                .strength_timeout_secs
413                .map_or(ds.timeout, |s| std::time::Duration::from_secs(s.max(1))),
414        }
415    };
416    let approval_wait = match approvals.wait.as_deref() {
417        Some(w) => wait(w)?,
418        None => d.approval_wait,
419    };
420    Ok(RunConfig {
421        max_turns: budget.max_turns.unwrap_or(d.max_turns).max(1),
422        max_tool_calls: budget.max_tool_calls.unwrap_or(d.max_tool_calls),
423        max_attempts: budget.max_attempts.unwrap_or(d.max_attempts).max(1),
424        best_of: budget.best_of.unwrap_or(d.best_of).clamp(1, 10),
425        allow: commands
426            .allow
427            .into_iter()
428            .filter(|p| !p.is_empty())
429            .collect(),
430        commit_per_task,
431        acceptance: acceptance.generate.unwrap_or(true),
432        held_out: acceptance.held_out.unwrap_or(true),
433        strength,
434        max_converge_rounds: converge.max_rounds.unwrap_or(d.max_converge_rounds),
435        monitor: monitor.enabled.unwrap_or(false),
436        review_required: review.required.unwrap_or(false),
437        frozen_additions,
438        format: format.map(|f| f.argv),
439        approval_wait,
440        attest,
441        instructions: context.instructions.unwrap_or(d.instructions),
442    })
443}
444
445#[cfg(test)]
446mod tests {
447    use super::*;
448
449    #[test]
450    fn agents_md_is_read_unless_configured_otherwise() {
451        assert_eq!(parse("").unwrap().instructions, ["AGENTS.md"]);
452        let c = parse("[context]\ninstructions = [\"AGENTS.md\", \"CLAUDE.md\"]\n").unwrap();
453        assert_eq!(c.instructions, ["AGENTS.md", "CLAUDE.md"]);
454        assert!(
455            parse("[context]\ninstructions = []\n")
456                .unwrap()
457                .instructions
458                .is_empty()
459        );
460        assert!(parse("[context]\ninstructions = [\"docs/AGENTS.md\"]\n").is_err());
461        assert!(parse("[context]\nfiles = []\n").is_err());
462    }
463
464    #[test]
465    fn attestations_are_on_and_signed_by_git_unless_configured() {
466        let d = parse("").unwrap_or_default().attest;
467        assert!(d.enabled && d.signer == Signer::Git && !d.sign_later);
468        let c = parse("[attest]\nsigner = \"sigstore\"\nsign_at = \"later\"\n[attest.sigstore]\nidentity = \"ada@example.com\"\nissuer = \"https://accounts.example\"\n")
469            .unwrap_or_default()
470            .attest;
471        assert_eq!((c.signer, c.sign_later), (Signer::Sigstore, true));
472        assert_eq!(c.sigstore_identity.as_deref(), Some("ada@example.com"));
473        assert!(parse("[attest]\nsigner = \"x509\"\n").is_err());
474        assert!(parse("[attest]\nsign_at = \"never\"\n").is_err());
475        assert!(parse("[attest]\nkey = \"x\"\n").is_err());
476    }
477
478    #[test]
479    fn an_approval_waits_seven_days_unless_configured() {
480        assert_eq!(
481            parse("").unwrap_or_default().approval_wait.as_secs(),
482            7 * 86_400
483        );
484        let w = |t: &str| {
485            parse(&format!("[approvals]\nwait = \"{t}\"\n")).map(|c| c.approval_wait.as_secs())
486        };
487        assert_eq!(w("30m"), Ok(1800));
488        assert_eq!(w("12h"), Ok(43_200));
489        for bad in ["0d", "7", "7w", "-1d", ""] {
490            assert!(w(bad).is_err(), "{bad}");
491        }
492    }
493
494    #[test]
495    fn strength_is_measured_by_default_and_bounded() {
496        let c = parse("").unwrap().strength;
497        assert!(c.enabled && c.min.is_none());
498        assert_eq!((c.near_miss, c.permitted), (2, 1));
499        let c = parse("[acceptance]\nmin_strength = 0.8\nnear_miss = 3\npermitted = 0\nstrength_timeout_secs = 60\n")
500            .unwrap()
501            .strength;
502        assert_eq!(c.min, Some(0.8));
503        assert_eq!((c.near_miss, c.permitted, c.timeout.as_secs()), (3, 0, 60));
504        assert!(
505            !parse("[acceptance]\nstrength = false\n")
506                .unwrap()
507                .strength
508                .enabled
509        );
510        for bad in ["min_strength = 1.5", "near_miss = 4", "permitted = 3"] {
511            assert!(parse(&format!("[acceptance]\n{bad}\n")).is_err(), "{bad}");
512        }
513    }
514
515    #[test]
516    fn held_out_is_on_by_default_and_can_be_switched_off() {
517        assert!(parse("").unwrap().held_out);
518        let c = parse("[acceptance]\nheld_out = false\n").unwrap();
519        assert!(!c.held_out && c.acceptance);
520        assert!(parse("[acceptance]\nheldout = true\n").is_err());
521    }
522
523    #[test]
524    fn defaults_and_overrides() {
525        assert_eq!(parse("").unwrap(), RunConfig::default());
526        let c =
527            parse("[budget]\nmax_turns = 5\n[commands]\nallow = [[\"cargo\", \"fmt\"]]\n").unwrap();
528        assert_eq!(c.max_turns, 5);
529        assert_eq!(c.max_tool_calls, 100);
530        assert_eq!(c.max_attempts, 2);
531        assert_eq!(
532            parse("[budget]\nmax_attempts = 0\n").unwrap().max_attempts,
533            1
534        );
535        assert_eq!(c.allow, [vec!["cargo".to_owned(), "fmt".to_owned()]]);
536        assert!(parse("[budget]\nmax_cost = 1\n").is_err());
537        assert!(!parse("[git]\ncommit = \"none\"\n").unwrap().commit_per_task);
538        assert!(parse("[git]\ncommit = \"always\"\n").is_err());
539        assert!(
540            parse("[tests]\nfrozen = \"additions\"\n")
541                .unwrap()
542                .frozen_additions
543        );
544        assert!(
545            !parse("[tests]\nfrozen = \"ask\"\n")
546                .unwrap()
547                .frozen_additions
548        );
549        assert!(parse("[tests]\nfrozen = \"anything\"\n").is_err());
550    }
551}
speccoder_runtime/substrate/run_config.rs

speccoder_runtime/substrate/
run_config.rs
