Skip to main content

mlua_swarm/store/trace/
mod.rs

1//! `RunTraceStore` — the persisted per-Run trace rail (`TraceEvent`
2//! stream) plus the normalized worker-stats types (`TokenUsage` /
3//! [`WorkerStats`]) shared by the trace rail and the
4//! [`crate::store::run::StepEntry`] step-stats extension.
5//!
6//! Two rails observe the same dispatch (issue: per-step run stats):
7//!
8//! - [`crate::store::run::StepEntry`] — the **terminal summary** of one
9//!   dispatched step (write-once, appended by the dispatcher after the
10//!   outcome is known; carries duration / usage / model / verdict).
11//! - [`TraceEvent`] (this module) — the **in-flight stream** of what is
12//!   happening (`core.step_dispatched`, `mw.long_hold_warn`, …),
13//!   append-only, per-Run, ordered by `seq`.
14//!
15//! Everything here is purely observational: a failed append must never
16//! fail the dispatch it observes (callers warn-and-swallow — the same
17//! fail-open convention as `EngineDispatcher::dispatch`'s
18//! `append_step_entry`). The naming is deliberately `Trace`, not `Log`:
19//! in the Rust ecosystem "log" collides with the `log`/`tracing`
20//! facade crates, and this rail is domain data, not process logging.
21//!
22//! Kinds are an **open set** of namespaced strings — writers may insert
23//! new kinds without a schema migration. Current namespaces:
24//!
25//! - `core.*` — engine/dispatcher (`run_started` / `step_dispatched` /
26//!   `step_completed` / `cancel_requested` / `run_finished`)
27//! - `mw.*` — middleware (`long_hold_warn`, …)
28//! - `worker.*` — adapter / worker self-reports
29//! - `ext.*` — future external writers (Lua flow, enhance flow, tools)
30//!
31//! Layering invariant (future `mlua-swarm-trace` crate split): this
32//! module must not depend on engine types — only `crate::types` ids and
33//! serde values.
34
35use crate::types::RunId;
36use async_trait::async_trait;
37use serde::{Deserialize, Serialize};
38use serde_json::Value;
39use std::collections::HashMap;
40use std::sync::{Arc, Mutex};
41use thiserror::Error;
42
43pub mod sqlite;
44pub use sqlite::SqliteRunTraceStore;
45
46// ──────────────────────────────────────────────────────────────────────────
47// Normalized worker stats (TokenUsage / WorkerStats)
48// ──────────────────────────────────────────────────────────────────────────
49
50/// Aggregated token usage for one worker attempt, normalized across
51/// worker kinds (agent-block `agent.run` return / subprocess declared
52/// mapping / operator self-report). Field names follow the Anthropic
53/// wire convention (`input_tokens` / `output_tokens`) that agent-block
54/// already normalizes OpenAI-style responses into.
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
56pub struct TokenUsage {
57    /// Prompt-side tokens consumed, summed across the attempt's turns.
58    pub input_tokens: u64,
59    /// Completion-side tokens produced, summed across the attempt's turns.
60    pub output_tokens: u64,
61    /// `input + output` (kept explicit because some producers report a
62    /// total that includes cache-read/creation tokens the two split
63    /// fields don't cover).
64    pub total_tokens: u64,
65}
66
67/// Normalized per-attempt worker statistics, reported by a worker
68/// boundary (spawner fold site / result captor / `POST
69/// /v1/worker/submit`) into the engine and folded into the terminal
70/// [`crate::store::run::StepEntry`] by the dispatcher.
71///
72/// The three named fields are the **closed schema** the engine knows;
73/// everything worker-kind-specific rides in [`Self::adapter_data`] as
74/// raw JSON the engine never interprets (capped at
75/// [`TRACE_PAYLOAD_CAP_BYTES`] on fold). Every field is optional —
76/// absence must never block a dispatch.
77#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
78pub struct WorkerStats {
79    /// Worker kind label (`"agent_block"` / `"subprocess"` /
80    /// `"operator"` / …) — set by whichever boundary constructed the
81    /// stats, since only that boundary knows its own kind.
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub worker_kind: Option<String>,
84    /// The model that served the attempt, when the boundary knows it
85    /// (subprocess: the rendered `{model}` placeholder; operator:
86    /// self-report; agent-block: spec-declared model if any).
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub model: Option<String>,
89    /// Normalized token usage, when the boundary can produce one.
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub usage: Option<TokenUsage>,
92    /// Number of LLM turns the attempt ran (agent-block `num_turns`).
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub num_turns: Option<u32>,
95    /// Worker-kind-specific raw payload (exit code, stderr tail, cache
96    /// token detail, …). Observational only — the engine stores it
97    /// verbatim (size-capped) and never branches on it.
98    #[serde(default, skip_serializing_if = "Option::is_none")]
99    pub adapter_data: Option<Value>,
100}
101
102impl WorkerStats {
103    /// `true` when no field carries information — callers skip
104    /// recording an all-empty stats value.
105    pub fn is_empty(&self) -> bool {
106        self.worker_kind.is_none()
107            && self.model.is_none()
108            && self.usage.is_none()
109            && self.num_turns.is_none()
110            && self.adapter_data.is_none()
111    }
112}
113
114// ──────────────────────────────────────────────────────────────────────────
115// TraceEvent / TraceQuery
116// ──────────────────────────────────────────────────────────────────────────
117
118/// Byte cap applied to [`TraceEvent::payload`] and
119/// [`WorkerStats::adapter_data`] before persisting. Oversized values are
120/// replaced by a truncation marker object (see [`cap_payload`]) — the
121/// trace rail is an observability artifact, not a blob store.
122pub const TRACE_PAYLOAD_CAP_BYTES: usize = 8 * 1024;
123
124/// Default per-Run retention ceiling — appends beyond this many events
125/// prune the oldest rows first.
126pub const DEFAULT_TRACE_MAX_EVENTS_PER_RUN: usize = 10_000;
127
128/// Default `list` page size when a query sets neither `limit` nor
129/// `latest`.
130pub const DEFAULT_TRACE_LIST_LIMIT: usize = 1_000;
131
132/// Well-known `TraceEvent.kind` values written by the engine itself.
133/// The kind axis is an open set — these constants exist so in-tree
134/// writers and tests agree on spelling, not to constrain writers.
135pub mod kind {
136    /// A Run began dispatching (server-side, once per kick).
137    pub const RUN_STARTED: &str = "core.run_started";
138    /// The dispatcher is about to spawn a step's worker.
139    pub const STEP_DISPATCHED: &str = "core.step_dispatched";
140    /// A step reached its terminal outcome (payload carries the status
141    /// label + timing summary; the authoritative record is the
142    /// `StepEntry` appended in the same breath).
143    pub const STEP_COMPLETED: &str = "core.step_completed";
144    /// A Run reached its terminal status (payload: `{"status": ...}`).
145    pub const RUN_FINISHED: &str = "core.run_finished";
146    /// Cancellation was requested for the Run.
147    pub const CANCEL_REQUESTED: &str = "core.cancel_requested";
148    /// `LongHoldMiddleware` observed a completion above its threshold.
149    pub const LONG_HOLD_WARN: &str = "mw.long_hold_warn";
150    /// A worker reported a degradation (mirrors the `DegradationEntry`
151    /// rail so the trace stream is self-contained).
152    pub const WORKER_DEGRADATION: &str = "worker.degradation";
153}
154
155/// One persisted trace event — a member of a Run's append-only stream.
156#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
157pub struct TraceEvent {
158    /// The Run this event belongs to.
159    #[schemars(with = "String")]
160    pub run_id: RunId,
161    /// Per-Run monotonically increasing ordering key, assigned by the
162    /// store at append time (1-based).
163    pub seq: u64,
164    /// Unix epoch milliseconds — when the event was recorded.
165    pub ts_ms: i64,
166    /// Namespaced kind string (open set; see the module doc).
167    pub kind: String,
168    /// The Blueprint step ref this event concerns, if any.
169    #[serde(default, skip_serializing_if = "Option::is_none")]
170    pub step_ref: Option<String>,
171    /// The attempt number this event concerns, if any.
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub attempt: Option<u32>,
174    /// Free-form JSON payload (capped at [`TRACE_PAYLOAD_CAP_BYTES`]).
175    pub payload: Value,
176}
177
178/// The caller-supplied half of a [`TraceEvent`] — the store assigns
179/// `seq` and `ts_ms` at append time.
180#[derive(Debug, Clone)]
181pub struct TraceEventDraft {
182    /// Namespaced kind string (open set).
183    pub kind: String,
184    /// The Blueprint step ref this event concerns, if any.
185    pub step_ref: Option<String>,
186    /// The attempt number this event concerns, if any.
187    pub attempt: Option<u32>,
188    /// Free-form JSON payload — capped by the store on append.
189    pub payload: Value,
190}
191
192/// Filter/paging parameters for [`RunTraceStore::list`]. All filters
193/// AND together; `latest` and `after` are mutually exclusive with
194/// `latest` winning (it answers "show me the tail" regardless of any
195/// cursor the caller also carried).
196#[derive(Debug, Clone, Default)]
197pub struct TraceQuery {
198    /// Forward-paging cursor: only events with `seq > after`.
199    pub after: Option<u64>,
200    /// Page size cap (defaults to [`DEFAULT_TRACE_LIST_LIMIT`]).
201    pub limit: Option<usize>,
202    /// Tail mode: return the LAST n matching events (still in ascending
203    /// `seq` order). Takes precedence over `after`/`limit`.
204    pub latest: Option<usize>,
205    /// Kind filters — an event matches when its kind equals, or starts
206    /// with, ANY entry (prefix match: `"mw."` matches every middleware
207    /// kind). Empty = no kind filter.
208    pub kinds: Vec<String>,
209    /// Exact `step_ref` filter.
210    pub step_ref: Option<String>,
211    /// Exact `attempt` filter.
212    pub attempt: Option<u32>,
213}
214
215impl TraceQuery {
216    /// Does `event` pass this query's kind/step/attempt filters
217    /// (paging axes excluded)?
218    fn matches(&self, event: &TraceEvent) -> bool {
219        if !self.kinds.is_empty()
220            && !self
221                .kinds
222                .iter()
223                .any(|k| event.kind == *k || event.kind.starts_with(k.as_str()))
224        {
225            return false;
226        }
227        if let Some(step_ref) = &self.step_ref {
228            if event.step_ref.as_deref() != Some(step_ref.as_str()) {
229                return false;
230            }
231        }
232        if let Some(attempt) = self.attempt {
233            if event.attempt != Some(attempt) {
234                return false;
235            }
236        }
237        true
238    }
239
240    /// Apply paging (latest wins over after/limit) to an ascending,
241    /// already-filtered event list.
242    fn page(&self, mut events: Vec<TraceEvent>) -> Vec<TraceEvent> {
243        if let Some(n) = self.latest {
244            let start = events.len().saturating_sub(n);
245            return events.split_off(start);
246        }
247        if let Some(after) = self.after {
248            events.retain(|e| e.seq > after);
249        }
250        let limit = self.limit.unwrap_or(DEFAULT_TRACE_LIST_LIMIT);
251        events.truncate(limit);
252        events
253    }
254}
255
256/// Replace an oversized payload with a truncation marker carrying the
257/// original size and a head excerpt, so a runaway writer cannot bloat
258/// the trace store. Values at or under [`TRACE_PAYLOAD_CAP_BYTES`] pass
259/// through unchanged.
260pub fn cap_payload(payload: Value) -> Value {
261    let serialized = payload.to_string();
262    if serialized.len() <= TRACE_PAYLOAD_CAP_BYTES {
263        return payload;
264    }
265    let head: String = serialized.chars().take(1024).collect();
266    serde_json::json!({
267        "truncated": true,
268        "size_bytes": serialized.len(),
269        "head": head,
270    })
271}
272
273/// Errors surfaced by a [`RunTraceStore`] implementation.
274#[derive(Debug, Error)]
275pub enum TraceStoreError {
276    /// Backend-specific failure.
277    #[error("other: {0}")]
278    Other(String),
279}
280
281// ──────────────────────────────────────────────────────────────────────────
282// RunTraceStore trait
283// ──────────────────────────────────────────────────────────────────────────
284
285/// Persistence interface for the per-Run trace stream.
286#[async_trait]
287pub trait RunTraceStore: Send + Sync {
288    /// Backend name — for diagnostics/logging.
289    fn name(&self) -> &str;
290
291    /// Append one event to `run_id`'s stream, assigning the next `seq`
292    /// and stamping `ts_ms`. The store caps `draft.payload` via
293    /// [`cap_payload`] and prunes the oldest rows beyond the per-Run
294    /// retention ceiling. Appending to an unknown `run_id` is legal —
295    /// the trace rail has no foreign-key coupling to `RunStore` (a
296    /// trace writer must never fail because Run-row creation raced it).
297    async fn append(
298        &self,
299        run_id: &RunId,
300        draft: TraceEventDraft,
301    ) -> Result<TraceEvent, TraceStoreError>;
302
303    /// List `run_id`'s events matching `query`, ascending by `seq`.
304    async fn list(
305        &self,
306        run_id: &RunId,
307        query: &TraceQuery,
308    ) -> Result<Vec<TraceEvent>, TraceStoreError>;
309
310    /// Delete every event belonging to `run_id`, returning the number
311    /// of deleted events. Deleting an unknown/empty Run is `Ok(0)`.
312    async fn delete_run(&self, run_id: &RunId) -> Result<u64, TraceStoreError>;
313}
314
315// ──────────────────────────────────────────────────────────────────────────
316// TraceHandle — the pervasive-insertion write port
317// ──────────────────────────────────────────────────────────────────────────
318
319/// A cheap, cloneable write handle binding one `run_id` to a
320/// [`RunTraceStore`] — the single port through which the dispatcher,
321/// middlewares (via `Engine::trace_handle`), server handlers, and any
322/// future writer append trace events. Appends are **best-effort**: a
323/// store failure is logged at `warn` and swallowed, never propagated
324/// (fail-open, matching the `append_step_entry` convention).
325#[derive(Clone)]
326pub struct TraceHandle {
327    run_id: RunId,
328    store: Arc<dyn RunTraceStore>,
329}
330
331impl TraceHandle {
332    /// Bind `run_id` to `store`.
333    pub fn new(run_id: RunId, store: Arc<dyn RunTraceStore>) -> Self {
334        Self { run_id, store }
335    }
336
337    /// The Run this handle appends into.
338    pub fn run_id(&self) -> &RunId {
339        &self.run_id
340    }
341
342    /// Best-effort append (see the struct doc). `kind` should follow
343    /// the namespaced open-set convention (`core.*` / `mw.*` /
344    /// `worker.*` / `ext.*`).
345    pub async fn append(
346        &self,
347        kind: &str,
348        step_ref: Option<&str>,
349        attempt: Option<u32>,
350        payload: Value,
351    ) {
352        let draft = TraceEventDraft {
353            kind: kind.to_string(),
354            step_ref: step_ref.map(str::to_string),
355            attempt,
356            payload,
357        };
358        if let Err(e) = self.store.append(&self.run_id, draft).await {
359            tracing::warn!(
360                run_id = %self.run_id,
361                kind = kind,
362                error = %e,
363                "TraceHandle::append failed (swallowed — trace is observational)"
364            );
365        }
366    }
367}
368
369impl std::fmt::Debug for TraceHandle {
370    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
371        f.debug_struct("TraceHandle")
372            .field("run_id", &self.run_id)
373            .field("store", &self.store.name())
374            .finish()
375    }
376}
377
378// ──────────────────────────────────────────────────────────────────────────
379// InMemoryRunTraceStore
380// ──────────────────────────────────────────────────────────────────────────
381
382#[derive(Default)]
383struct TraceInner {
384    /// Per-Run ascending event lists.
385    events: HashMap<RunId, Vec<TraceEvent>>,
386    /// Per-Run next `seq` — kept separately from `events.len()` because
387    /// retention pruning removes head entries without recycling seqs.
388    next_seq: HashMap<RunId, u64>,
389}
390
391/// Process-volatile [`RunTraceStore`] — the default when no persistent
392/// backend is wired.
393pub struct InMemoryRunTraceStore {
394    inner: Mutex<TraceInner>,
395    max_events_per_run: usize,
396}
397
398impl InMemoryRunTraceStore {
399    /// Create an empty store with the default retention ceiling.
400    pub fn new() -> Self {
401        Self {
402            inner: Mutex::new(TraceInner::default()),
403            max_events_per_run: DEFAULT_TRACE_MAX_EVENTS_PER_RUN,
404        }
405    }
406
407    /// Create an empty store with a custom per-Run retention ceiling
408    /// (tests).
409    pub fn with_max_events_per_run(max: usize) -> Self {
410        Self {
411            inner: Mutex::new(TraceInner::default()),
412            max_events_per_run: max,
413        }
414    }
415}
416
417impl Default for InMemoryRunTraceStore {
418    fn default() -> Self {
419        Self::new()
420    }
421}
422
423/// Unix epoch milliseconds now — trace events are sub-second dense, so
424/// the store stamps millis (the coarser `now_unix` seconds stay on the
425/// pre-existing `StepEntry.at` / `RunRecord` fields).
426pub(crate) fn now_unix_ms() -> i64 {
427    use std::time::{SystemTime, UNIX_EPOCH};
428    SystemTime::now()
429        .duration_since(UNIX_EPOCH)
430        .map(|d| d.as_millis() as i64)
431        .unwrap_or(0)
432}
433
434#[async_trait]
435impl RunTraceStore for InMemoryRunTraceStore {
436    fn name(&self) -> &str {
437        "in-memory"
438    }
439
440    async fn append(
441        &self,
442        run_id: &RunId,
443        draft: TraceEventDraft,
444    ) -> Result<TraceEvent, TraceStoreError> {
445        let mut inner = self.inner.lock().unwrap();
446        let seq_slot = inner.next_seq.entry(run_id.clone()).or_insert(0);
447        *seq_slot += 1;
448        let event = TraceEvent {
449            run_id: run_id.clone(),
450            seq: *seq_slot,
451            ts_ms: now_unix_ms(),
452            kind: draft.kind,
453            step_ref: draft.step_ref,
454            attempt: draft.attempt,
455            payload: cap_payload(draft.payload),
456        };
457        let list = inner.events.entry(run_id.clone()).or_default();
458        list.push(event.clone());
459        if list.len() > self.max_events_per_run {
460            let overflow = list.len() - self.max_events_per_run;
461            list.drain(..overflow);
462        }
463        Ok(event)
464    }
465
466    async fn list(
467        &self,
468        run_id: &RunId,
469        query: &TraceQuery,
470    ) -> Result<Vec<TraceEvent>, TraceStoreError> {
471        let inner = self.inner.lock().unwrap();
472        let events: Vec<TraceEvent> = inner
473            .events
474            .get(run_id)
475            .map(|list| list.iter().filter(|e| query.matches(e)).cloned().collect())
476            .unwrap_or_default();
477        Ok(query.page(events))
478    }
479
480    async fn delete_run(&self, run_id: &RunId) -> Result<u64, TraceStoreError> {
481        let mut inner = self.inner.lock().unwrap();
482        inner.next_seq.remove(run_id);
483        Ok(inner
484            .events
485            .remove(run_id)
486            .map(|list| list.len() as u64)
487            .unwrap_or(0))
488    }
489}
490
491// ──────────────────────────────────────────────────────────────────────────
492// tests
493// ──────────────────────────────────────────────────────────────────────────
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498    use serde_json::json;
499
500    fn rid(s: &str) -> RunId {
501        RunId::parse(s).unwrap()
502    }
503
504    fn draft(kind: &str, step_ref: Option<&str>, attempt: Option<u32>) -> TraceEventDraft {
505        TraceEventDraft {
506            kind: kind.to_string(),
507            step_ref: step_ref.map(str::to_string),
508            attempt,
509            payload: json!({"k": kind}),
510        }
511    }
512
513    #[tokio::test]
514    async fn append_assigns_monotonic_seq_per_run() {
515        let s = InMemoryRunTraceStore::new();
516        let e1 = s
517            .append(&rid("R-1"), draft("core.run_started", None, None))
518            .await
519            .unwrap();
520        let e2 = s
521            .append(
522                &rid("R-1"),
523                draft("core.step_dispatched", Some("w"), Some(1)),
524            )
525            .await
526            .unwrap();
527        let other = s
528            .append(&rid("R-2"), draft("core.run_started", None, None))
529            .await
530            .unwrap();
531        assert_eq!(e1.seq, 1);
532        assert_eq!(e2.seq, 2);
533        assert_eq!(other.seq, 1, "seq is per-Run, not global");
534        assert!(e1.ts_ms > 0);
535    }
536
537    #[tokio::test]
538    async fn list_filters_by_kind_prefix_step_and_attempt() {
539        let s = InMemoryRunTraceStore::new();
540        let r = rid("R-1");
541        s.append(&r, draft("core.run_started", None, None))
542            .await
543            .unwrap();
544        s.append(&r, draft("core.step_dispatched", Some("a"), Some(1)))
545            .await
546            .unwrap();
547        s.append(&r, draft("mw.long_hold_warn", Some("a"), Some(1)))
548            .await
549            .unwrap();
550        s.append(&r, draft("core.step_completed", Some("b"), Some(2)))
551            .await
552            .unwrap();
553
554        let mw = s
555            .list(
556                &r,
557                &TraceQuery {
558                    kinds: vec!["mw.".into()],
559                    ..Default::default()
560                },
561            )
562            .await
563            .unwrap();
564        assert_eq!(mw.len(), 1);
565        assert_eq!(mw[0].kind, "mw.long_hold_warn");
566
567        let step_a = s
568            .list(
569                &r,
570                &TraceQuery {
571                    step_ref: Some("a".into()),
572                    ..Default::default()
573                },
574            )
575            .await
576            .unwrap();
577        assert_eq!(step_a.len(), 2);
578
579        let attempt2 = s
580            .list(
581                &r,
582                &TraceQuery {
583                    attempt: Some(2),
584                    ..Default::default()
585                },
586            )
587            .await
588            .unwrap();
589        assert_eq!(attempt2.len(), 1);
590        assert_eq!(attempt2[0].step_ref.as_deref(), Some("b"));
591    }
592
593    #[tokio::test]
594    async fn list_paging_after_and_latest() {
595        let s = InMemoryRunTraceStore::new();
596        let r = rid("R-1");
597        for i in 0..5 {
598            s.append(&r, draft(&format!("core.e{i}"), None, None))
599                .await
600                .unwrap();
601        }
602
603        let after = s
604            .list(
605                &r,
606                &TraceQuery {
607                    after: Some(3),
608                    ..Default::default()
609                },
610            )
611            .await
612            .unwrap();
613        assert_eq!(after.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![4, 5]);
614
615        let latest = s
616            .list(
617                &r,
618                &TraceQuery {
619                    latest: Some(2),
620                    // latest must win over a cursor the caller also set.
621                    after: Some(1),
622                    ..Default::default()
623                },
624            )
625            .await
626            .unwrap();
627        assert_eq!(latest.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![4, 5]);
628
629        let limited = s
630            .list(
631                &r,
632                &TraceQuery {
633                    limit: Some(2),
634                    ..Default::default()
635                },
636            )
637            .await
638            .unwrap();
639        assert_eq!(
640            limited.iter().map(|e| e.seq).collect::<Vec<_>>(),
641            vec![1, 2]
642        );
643    }
644
645    #[tokio::test]
646    async fn retention_prunes_oldest_keeping_seq() {
647        let s = InMemoryRunTraceStore::with_max_events_per_run(3);
648        let r = rid("R-1");
649        for i in 0..5 {
650            s.append(&r, draft(&format!("core.e{i}"), None, None))
651                .await
652                .unwrap();
653        }
654        let all = s.list(&r, &TraceQuery::default()).await.unwrap();
655        assert_eq!(all.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![3, 4, 5]);
656        // A later append keeps counting up — pruning never recycles seqs.
657        let e6 = s.append(&r, draft("core.e5", None, None)).await.unwrap();
658        assert_eq!(e6.seq, 6);
659    }
660
661    #[tokio::test]
662    async fn delete_run_removes_stream() {
663        let s = InMemoryRunTraceStore::new();
664        let r = rid("R-1");
665        s.append(&r, draft("core.run_started", None, None))
666            .await
667            .unwrap();
668        s.append(&r, draft("core.run_finished", None, None))
669            .await
670            .unwrap();
671        assert_eq!(s.delete_run(&r).await.unwrap(), 2);
672        assert!(s.list(&r, &TraceQuery::default()).await.unwrap().is_empty());
673        assert_eq!(s.delete_run(&r).await.unwrap(), 0, "double delete is Ok(0)");
674    }
675
676    #[tokio::test]
677    async fn oversized_payload_is_truncated_with_marker() {
678        let s = InMemoryRunTraceStore::new();
679        let r = rid("R-1");
680        let big = "x".repeat(TRACE_PAYLOAD_CAP_BYTES + 100);
681        let e = s
682            .append(
683                &r,
684                TraceEventDraft {
685                    kind: "worker.output".into(),
686                    step_ref: None,
687                    attempt: None,
688                    payload: json!({"blob": big}),
689                },
690            )
691            .await
692            .unwrap();
693        assert_eq!(e.payload.get("truncated"), Some(&json!(true)));
694        assert!(e.payload.get("size_bytes").is_some());
695    }
696
697    #[tokio::test]
698    async fn trace_handle_appends_best_effort() {
699        let store: Arc<dyn RunTraceStore> = Arc::new(InMemoryRunTraceStore::new());
700        let handle = TraceHandle::new(rid("R-1"), store.clone());
701        handle
702            .append(kind::STEP_DISPATCHED, Some("w"), Some(1), json!({}))
703            .await;
704        let events = store
705            .list(&rid("R-1"), &TraceQuery::default())
706            .await
707            .unwrap();
708        assert_eq!(events.len(), 1);
709        assert_eq!(events[0].kind, kind::STEP_DISPATCHED);
710    }
711
712    #[test]
713    fn worker_stats_is_empty_reflects_fields() {
714        assert!(WorkerStats::default().is_empty());
715        let stats = WorkerStats {
716            usage: Some(TokenUsage {
717                input_tokens: 1,
718                output_tokens: 2,
719                total_tokens: 3,
720            }),
721            ..Default::default()
722        };
723        assert!(!stats.is_empty());
724    }
725}