dekopon-shell 0.18.0

Sandboxed bash-flavored script interpreter whose commands dispatch to Dekopon capabilities
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
//! What the per-command spans carry.
//!
//! These assert on real emitted `tracing` output rather than on the helpers in the parent module,
//! because what a span records is a property of the call site: a field recorded wrongly, or not at
//! all, in `run_argv` would leave every helper here passing.

use std::{
    collections::BTreeMap,
    fmt,
    sync::{Arc, Mutex},
};

use tracing::{
    Subscriber,
    field::{Field, Visit},
    span,
};
use tracing_subscriber::{
    Layer,
    layer::{Context, SubscriberExt},
    registry::LookupSpan,
};

use crate::{Interpreter, Limits, ScriptOutcome, interp::tests::Fixture};

use super::{CONTROL_WORDS, SCRIPT_SPAN};

/// One span, flattened to the strings a remote collector would receive.
#[derive(Clone, Debug)]
struct Captured {
    /// Span name.
    span: Option<String>,
    /// Enclosing span names, innermost first.
    parents: Vec<String>,
    fields: BTreeMap<String, String>,
}

impl Captured {
    fn field(&self, name: &str) -> Option<&str> {
        self.fields.get(name).map(String::as_str)
    }
}

/// Collects every field of a span as its rendered string.
///
/// Every value is rendered to the text a collector would hold, whatever type it was recorded with:
/// integers such as the `.bytes` totals arrive through `record_debug`.
#[derive(Default)]
struct Fields(BTreeMap<String, String>);

impl Visit for Fields {
    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
        self.0.insert(field.name().to_owned(), format!("{value:?}"));
    }

    fn record_str(&mut self, field: &Field, value: &str) {
        self.0.insert(field.name().to_owned(), value.to_owned());
    }
}

/// A subscriber layer that records every span for later inspection.
struct CaptureLayer {
    spans: Arc<Mutex<Vec<Captured>>>,
}

impl<S> Layer<S> for CaptureLayer
where
    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
    fn on_new_span(
        &self,
        attributes: &span::Attributes<'_>,
        id: &span::Id,
        context: Context<'_, S>,
    ) {
        let mut fields = Fields::default();
        attributes.record(&mut fields);
        if let Some(span) = context.span(id) {
            span.extensions_mut().insert(fields);
        }
    }

    /// Fields recorded after creation — arguments, stdin, output, the exit code, and the outcome
    /// — arrive here.
    fn on_record(&self, id: &span::Id, values: &span::Record<'_>, context: Context<'_, S>) {
        if let Some(span) = context.span(id) {
            let mut extensions = span.extensions_mut();
            if let Some(fields) = extensions.get_mut::<Fields>() {
                values.record(fields);
            }
        }
    }

    fn on_close(&self, id: span::Id, context: Context<'_, S>) {
        let Some(span) = context.span(&id) else {
            return;
        };
        let fields = span
            .extensions()
            .get::<Fields>()
            .map(|fields| fields.0.clone())
            .unwrap_or_default();
        self.spans.lock().expect("span lock").push(Captured {
            span: Some(span.name().to_owned()),
            parents: span
                .scope()
                .skip(1)
                .map(|parent| parent.name().to_owned())
                .collect(),
            fields,
        });
    }
}

/// Everything one script emitted.
struct Telemetry {
    outcome: ScriptOutcome,
    spans: Vec<Captured>,
}

impl Telemetry {
    /// Every `shell.command` span, as `(kind, name)` pairs.
    ///
    /// Spans are captured on close, so these are in completion order: a command nested inside a
    /// shell function closes before the function does.
    fn commands(&self) -> Vec<(&str, &str)> {
        self.spans
            .iter()
            .filter(|span| span.span.as_deref() == Some("shell.command"))
            .map(|span| {
                (
                    span.field("shell.command.kind").unwrap_or("<missing>"),
                    span.field("shell.command.name").unwrap_or("<missing>"),
                )
            })
            .collect()
    }

    /// Every `shell.command` span one command word opened, in completion order.
    fn command_spans(&self, name: &str) -> Vec<&Captured> {
        self.spans
            .iter()
            .filter(|span| {
                span.span.as_deref() == Some("shell.command")
                    && span.field("shell.command.name") == Some(name)
            })
            .collect()
    }
}

/// Runs one script under a capturing subscriber scoped to this thread.
fn capture(script: &str) -> Telemetry {
    capture_with(script, Limits::default(), false)
}

/// Runs one script, optionally inside an enclosing span standing in for the runner's own.
fn capture_with(script: &str, limits: Limits, enclose: bool) -> Telemetry {
    let spans = Arc::new(Mutex::new(Vec::new()));
    let subscriber = tracing_subscriber::registry().with(CaptureLayer {
        spans: Arc::clone(&spans),
    });

    // Thread-local rather than global, so these tests stay independent under the default
    // parallel test harness.
    let outcome = tracing::subscriber::with_default(subscriber, || {
        // Created *inside* `with_default`: a span built while no subscriber is installed is
        // disabled forever, and entering it would prove nothing about nesting.
        let enclosing = enclose.then(|| tracing::info_span!("caller.enclosing"));
        let _entered = enclosing.as_ref().map(tracing::Span::enter);
        Interpreter::new(limits).run(script, &Fixture::default())
    });

    let spans = spans.lock().expect("span lock").clone();
    Telemetry { outcome, spans }
}

/// Asserts one recorded attribute and the byte total recorded beside it.
fn assert_recorded(span: &Captured, field: &str, recorded: &str, total: usize) {
    assert_eq!(span.field(field), Some(recorded), "{field}");
    let bytes = format!("{field}.bytes");
    assert_eq!(
        span.field(&bytes),
        Some(total.to_string().as_str()),
        "{bytes}"
    );
}

#[test]
fn every_command_produces_exactly_one_span() {
    let telemetry =
        capture("greet() { echo hi; }\ngreet\njq -n 1\nprobe upper --text two\nnosuchcommand\n:");

    // One span per command word actually executed. `greet`'s body runs `echo`, so the function
    // call and the command inside it are both here — which is the point of instrumenting the one
    // seam every command passes through.
    assert_eq!(
        telemetry.commands(),
        vec![
            // `echo` runs inside `greet`, so it closes first. Spans are captured on close, which
            // makes this completion order rather than start order.
            ("builtin", "echo"),
            ("function", "greet"),
            ("builtin", "jq"),
            ("provider-command", "probe"),
            ("not-found", "nosuchcommand"),
            ("control", ":"),
        ]
    );

    let spans = telemetry
        .spans
        .iter()
        .filter(|span| span.span.as_deref() == Some("shell.command"))
        .count();
    assert_eq!(spans, 6);
}

#[test]
fn a_span_carries_the_outcome_exit_code_and_argument_count() {
    let telemetry = capture("echo one two three");
    let span = telemetry
        .spans
        .iter()
        .find(|span| span.span.as_deref() == Some("shell.command"))
        .expect("a command span");

    assert_eq!(span.field("shell.command.name"), Some("echo"));
    assert_eq!(span.field("shell.command.kind"), Some("builtin"));
    assert_eq!(span.field("shell.command.argument_count"), Some("3"));
    assert_eq!(
        span.field("shell.command.arguments"),
        Some(r#"["one","two","three"]"#)
    );
    assert_eq!(span.field("shell.command.exit_code"), Some("0"));
    assert_eq!(span.field("outcome"), Some("succeeded"));
}

#[test]
fn a_denied_capability_is_not_flattened_into_a_generic_failure() {
    // A refusal, a provider that ran and errored, and a capability that is not reachable are three
    // different operational stories; `CapabilityCallResult` keeps them apart and so must this.
    for (script, outcome, exit_code) in [
        ("probe upper --text hi", "succeeded", "0"),
        ("probe broken", "failed", "1"),
        ("probe denied", "denied", "126"),
        // A proposal for a capability this session was not granted, and a word nothing provides,
        // land on the same code and the same label.
        ("probe ungranted", "not-found", "127"),
        ("nosuchcommand", "not-found", "127"),
    ] {
        let telemetry = capture(script);
        let completed = telemetry
            .spans
            .iter()
            .find(|span| span.span.as_deref() == Some("shell.command"))
            .unwrap_or_else(|| panic!("{script}: a completed span"));
        assert_eq!(completed.field("outcome"), Some(outcome), "{script}");
        assert_eq!(
            completed.field("shell.command.exit_code"),
            Some(exit_code),
            "{script}"
        );
    }
}

#[test]
fn a_refused_word_reports_the_reason_it_aborted_the_script() {
    let telemetry = capture("eval 'echo hi'");
    let completed = telemetry
        .spans
        .iter()
        .find(|span| span.span.as_deref() == Some("shell.command"))
        .expect("a shell.command span");

    assert_eq!(completed.field("shell.command.kind"), Some("rejected"));
    // The word comes from this crate's own refusal table, so naming it exports nothing the script
    // authored.
    assert_eq!(completed.field("shell.command.name"), Some("eval"));
    assert_eq!(completed.field("outcome"), Some("rejected"));
    assert_eq!(telemetry.outcome.exit_code.get(), 2);
}

#[test]
fn an_exhausted_budget_is_reported_as_a_limit_rather_than_a_failure() {
    // The capability ceiling is the one a *command* trips. The step budget is charged between
    // statements, so it ends the script without any command span ever seeing it.
    let telemetry = capture_with(
        "while true; do probe upper --text x; done",
        Limits {
            max_capability_calls: 2,
            ..Limits::default()
        },
        false,
    );
    let completed = telemetry
        .spans
        .iter()
        .rfind(|span| span.span.as_deref() == Some("shell.command"))
        .expect("a shell.command span");
    assert_eq!(completed.field("outcome"), Some("limit-exceeded"));
}

#[test]
fn a_command_span_records_its_arguments_stdin_and_output() {
    // One trace, complete: the argv a command word ran with, the value piped into it, and what it
    // produced are the record of what the agent did, not just the word it typed.
    let telemetry = capture("probe upper --text hello\necho piped | probe upper -");
    let probes = telemetry.command_spans("probe");
    let [flag, piped] = probes.as_slice() else {
        panic!("two probe spans: {probes:?}");
    };

    let arguments = r#"["upper","--text","hello"]"#;
    assert_recorded(flag, "shell.command.arguments", arguments, arguments.len());
    // The capability's result, not an echo of the argv: the fixture uppercases.
    let output = r#"{"text":"HELLO"}"#;
    assert_recorded(flag, "shell.command.output", output, output.len());
    // Nothing was piped, which is a different fact from an empty string being piped.
    assert_eq!(flag.field("shell.command.stdin"), None);
    assert_eq!(flag.field("shell.command.stdin.bytes"), None);

    let arguments = r#"["upper","-"]"#;
    assert_recorded(piped, "shell.command.arguments", arguments, arguments.len());
    assert_recorded(piped, "shell.command.stdin", "piped", "piped".len());
    let output = r#"{"text":"PIPED"}"#;
    assert_recorded(piped, "shell.command.output", output, output.len());
}

#[test]
fn an_oversized_attribute_keeps_a_4096_byte_head_a_marker_and_its_full_length() {
    // Bound an attribute's size, never the span count. The cap and the marker are the documented
    // contract an operator's queries rely on, so they are spelled out here rather than borrowed
    // from the implementation.
    const CAP: usize = 4096;
    const MARKER: &str = "…[truncated]";

    let payload = "x".repeat(CAP + 904);
    let telemetry = capture(&format!(
        "probe upper --text {payload}\necho {payload} | probe upper -"
    ));
    let probes = telemetry.command_spans("probe");
    let [flag, piped] = probes.as_slice() else {
        panic!("two probe spans: {probes:?}");
    };

    let arguments = format!(r#"["upper","--text","{payload}"]"#);
    let output = format!(r#"{{"text":"{}"}}"#, payload.to_uppercase());
    for (span, field, full) in [
        (flag, "shell.command.arguments", &arguments),
        (flag, "shell.command.output", &output),
        (piped, "shell.command.stdin", &payload),
        (piped, "shell.command.output", &output),
    ] {
        // Every byte here is ASCII, so the head ends on a character boundary at exactly the cap.
        let head = format!("{}{MARKER}", &full[..CAP]);
        assert_recorded(span, field, &head, full.len());
    }

    // A value of exactly the cap is whole: nothing was cut, so nothing says it was.
    let exact = "x".repeat(CAP - r#"["upper","--text",""]"#.len());
    let telemetry = capture(&format!("probe upper --text {exact}"));
    let probes = telemetry.command_spans("probe");
    let [span] = probes.as_slice() else {
        panic!("one probe span: {probes:?}");
    };
    let arguments = format!(r#"["upper","--text","{exact}"]"#);
    assert_eq!(arguments.len(), CAP);
    assert_recorded(span, "shell.command.arguments", &arguments, CAP);
}

#[test]
fn a_model_authored_command_word_is_recorded_verbatim() {
    // A shell function's name and an unresolved word are both whatever the script's author typed —
    // and the author is usually the model, which is exactly why they belong in the trace. An
    // operator reading a session has to see the word the model reached for, including the one that
    // resolved to nothing.
    let telemetry = capture("model_helper() { echo hi; }\nmodel_helper\nmodel_typo");

    assert_eq!(
        telemetry.commands(),
        vec![
            // Completion order: the body of `model_helper` closes before the function itself.
            ("builtin", "echo"),
            ("function", "model_helper"),
            ("not-found", "model_typo"),
        ]
    );
}

/// A capability identifier typed as a command word is an unknown word, and the trace says so.
///
/// There is no separate kind for it, granted or not. The word is on the span and its arguments
/// beside it, so an operator still sees exactly what the model reached for.
#[test]
fn a_capability_shaped_word_is_recorded_as_not_found_with_its_arguments() {
    let telemetry = capture("cli-probe.upper --text hi\nwikipedia_page --title x");

    assert_eq!(
        telemetry.commands(),
        vec![
            ("not-found", "cli-probe.upper"),
            ("not-found", "wikipedia_page"),
        ]
    );
    let spans = telemetry.command_spans("wikipedia_page");
    let [span] = spans.as_slice() else {
        panic!("one wikipedia_page span: {spans:?}");
    };
    let arguments = r#"["--title","x"]"#;
    assert_recorded(span, "shell.command.arguments", arguments, arguments.len());
    assert_eq!(span.field("shell.command.exit_code"), Some("127"));
}

#[test]
fn xargs_records_every_command_it_actually_drove() {
    // One script word that maps a command over three items really did run three commands, so a
    // trace that showed one would be describing a script nobody wrote.
    // The list is built through `jq` rather than `echo`, because `echo` produces one string and
    // `xargs` would then have a single element to map over, passing for the wrong reason.
    let telemetry =
        capture("probe object --a a --b b --c c | jq '[.a,.b,.c]' | xargs probe upper --text");

    // The producer of the list is one `probe` span beside the `xargs` one; each element adds one
    // more nested *inside* it, so the relationship between the one script word and the commands it
    // drove survives into the trace.
    let probes = telemetry.command_spans("probe");
    assert_eq!(probes.len(), 4, "the producer plus one per element");
    let nested = probes
        .iter()
        .filter(|span| span.parents.iter().any(|parent| parent == "shell.command"))
        .map(|span| span.field("shell.command.arguments").unwrap_or("<missing>"))
        .collect::<Vec<_>>();
    // Each drove its own argv, element appended, not one command for the whole list.
    assert_eq!(
        nested,
        vec![
            r#"["upper","--text","a"]"#,
            r#"["upper","--text","b"]"#,
            r#"["upper","--text","c"]"#,
        ]
    );
}

#[test]
fn command_spans_nest_under_the_callers_active_span() {
    // An embedder enters its parent span and calls the interpreter on the same thread;
    // nesting needs no propagation code. This pins that shared interpreter contract.
    let telemetry = capture_with("echo hi", Limits::default(), true);

    let span = telemetry
        .spans
        .iter()
        .find(|span| span.span.as_deref() == Some("shell.command"))
        .expect("a command span");
    // The script's own span sits between the command and the caller's: the totals need a home that
    // costs the same whatever the script did, and the caller's span is not this crate's to write to.
    assert_eq!(
        span.parents,
        vec![SCRIPT_SPAN.to_owned(), "caller.enclosing".to_owned()]
    );
}

#[test]
fn one_script_span_carries_the_totals_for_the_whole_run() {
    let telemetry = capture("greet() { echo hi; }\ngreet\nnosuchcommand\nprobe upper --text two");

    let script = telemetry
        .spans
        .iter()
        .find(|span| span.span.as_deref() == Some(SCRIPT_SPAN))
        .expect("one script span");
    // `greet`, the `echo` inside it, `nosuchcommand`, and the provider command.
    assert_eq!(script.field("shell.script.commands"), Some("4"));
    assert_eq!(script.field("shell.script.capability_commands"), Some("1"));
    assert_eq!(script.field("shell.script.failed_commands"), Some("1"));

    let scripts = telemetry
        .spans
        .iter()
        .filter(|span| span.span.as_deref() == Some(SCRIPT_SPAN))
        .count();
    assert_eq!(scripts, 1, "one span per run, not one per statement");
}

#[test]
fn a_loop_heavy_script_still_spans_every_command_it_ran() {
    // A model-authored `while` loop is bounded only by the step budget, and every command word it
    // runs gets its span: an attribute may be truncated, a span is never dropped. A trace that
    // thinned out after the first few hundred commands would answer "what did this agent do" with
    // "up to here", which is the one answer the operator's trace may not give.
    let commands = 300;
    let telemetry = capture(&format!(
        "i=0\nwhile [ $i -lt {commands} ]; do echo x; i=$(( i + 1 )); done"
    ));

    let script = telemetry
        .spans
        .iter()
        .find(|span| span.span.as_deref() == Some(SCRIPT_SPAN))
        .expect("one script span");
    let total = script
        .field("shell.script.commands")
        .and_then(|value| value.parse::<usize>().ok())
        .expect("a command total");
    assert!(
        total > commands,
        "the loop must run more commands than any plausible cap for this to prove anything: {total}"
    );

    let spans = telemetry
        .spans
        .iter()
        .filter(|span| span.span.as_deref() == Some("shell.command"))
        .count();
    assert_eq!(spans, total, "one span per command word, all the way down");

    // The one failure is the `[` that finally reports false and ends the loop, counted like any
    // other non-zero status.
    assert_eq!(script.field("shell.script.failed_commands"), Some("1"));
}

#[test]
fn control_words_and_their_dispatcher_agree() {
    // `run_argv` classifies a control word from `CONTROL_WORDS` and only then lets
    // `run_control_word` execute it, so a word dropped from the list stops running and says
    // "command not found" instead. That is the direction this covers.
    //
    // The reverse — an arm added to `run_control_word` but not to the list — is caught by
    // construction rather than here: such a word reaches `dispatch::resolve`, and no control word
    // is in the builtin registry or the rejection table, so it also lands on "command not found"
    // rather than silently doing something else. Both directions fail closed and loudly.
    for word in CONTROL_WORDS {
        let outcome = capture(word).outcome;
        assert!(
            !outcome.output.contains("command not found"),
            "{word}: {}",
            outcome.output
        );
    }

    // The assertion above is only meaningful because an unknown word really does say this.
    assert!(
        capture("definitelynotacontrolword")
            .outcome
            .output
            .contains("command not found")
    );
}