sprawling 0.0.12

A local multi-agent harness for persistent agent teams: long-running project work, agent communication, and experiments in coordination and social simulation.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (c) 2026 2youg1 and the sprawling contributors

//! Whether a served city that just ended is left down, raised again, or
//! handed to the person (`crates/sprawling/spec/Supervising.lean` ยง8-109; the budget's
//! properties are proved in `crates/sprawling/spec/Supervising.lean`).
//!
//! The decision alone: no process, no clock. `children` runs the
//! processes, samples the time and reads a child's exit code as served
//! or failed; who closed the city and how is written in the child's own
//! handoff, which this module never needs to read.

mod children;

pub use children::{Child, Console, Ended, Window, supervise};

use kernel::{AxError, TimeMs};

/// How long a crash counts against the budget.
pub(crate) const CRASH_WINDOW_MS: u64 = 60_000;
/// How many crashes inside the window make the city degraded.
pub(crate) const CRASH_LIMIT: usize = 3;

/// The crashes still inside the window, oldest first; never more than
/// `CRASH_LIMIT`, because the one that reaches it ends the budget.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CrashBudget {
    crashes: Vec<TimeMs>,
}

/// What the supervisor does after a served city ended.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Next {
    /// The person closed the city; supervising ends with it.
    Stop,
    /// A crash inside the budget: resume, then serve again.
    Restart(CrashBudget),
    /// The budget is spent: nothing restarts until the person lifts it.
    Degraded { crashes: usize, cause: String },
}

impl CrashBudget {
    /// A budget with no crash counted, which is also what lifting a
    /// degraded city starts from.
    pub(crate) fn fresh() -> Self {
        Self {
            crashes: Vec::with_capacity(CRASH_LIMIT),
        }
    }

    /// Spends this budget on one ending observed at `at`.
    pub(crate) fn after(mut self, served: &Result<(), AxError>, at: TimeMs) -> Next {
        match served {
            Ok(()) => Next::Stop,
            Err(failure) => {
                self.crashes
                    .retain(|crash| at.value().saturating_sub(crash.value()) < CRASH_WINDOW_MS);
                self.crashes.push(at);
                if self.crashes.len() >= CRASH_LIMIT {
                    Next::Degraded {
                        crashes: self.crashes.len(),
                        cause: failure.to_string(),
                    }
                } else {
                    Next::Restart(self)
                }
            }
        }
    }
}

#[cfg(test)]
#[allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::arithmetic_side_effects,
    reason = "test code"
)]
mod tests {
    use super::{CrashBudget, Next};
    use kernel::{AxCode, AxError, TimeMs};

    fn crash() -> Result<(), AxError> {
        Err(AxError::failure(
            AxCode::StorageFatal,
            "serve a supervised city",
            "exit code: 101",
        )
        .with_recovery("start it again"))
    }

    /// Feeds crashes at the given seconds and returns every verdict.
    fn verdicts(seconds: &[u64]) -> Vec<Next> {
        let mut budget = CrashBudget::fresh();
        let mut seen = Vec::new();
        for s in seconds {
            let next = budget.clone().after(&crash(), TimeMs::new(s * 1_000));
            if let Next::Restart(kept) = &next {
                budget = kept.clone();
            }
            seen.push(next);
        }
        seen
    }

    fn restart(seconds: &[u64]) -> Next {
        let mut budget = CrashBudget::fresh();
        budget.crashes = seconds.iter().map(|s| TimeMs::new(s * 1_000)).collect();
        Next::Restart(budget)
    }

    #[test]
    fn three_crashes_inside_a_minute_degrade_and_older_ones_expire() {
        let degraded = Next::Degraded {
            crashes: 3,
            cause: crash().unwrap_err().to_string(),
        };
        assert_eq!(
            (
                verdicts(&[0, 10, 20]),
                verdicts(&[0, 10, 60]),
                CrashBudget::fresh().after(&Ok(()), TimeMs::new(5)),
            ),
            (
                vec![restart(&[0]), restart(&[0, 10]), degraded],
                vec![restart(&[0]), restart(&[0, 10]), restart(&[10, 60])],
                Next::Stop,
            )
        );
    }
}