mlua_swarm/store/trace/mod.rs
1//! `RunTraceStore` — the persisted per-Run trace rail (`TraceEvent`
2//! stream) plus the normalized worker-stats types (`TokenUsage` /
3//! [`WorkerStats`]) shared by the trace rail and the
4//! [`crate::store::run::StepEntry`] step-stats extension.
5//!
6//! Two rails observe the same dispatch (issue: per-step run stats):
7//!
8//! - [`crate::store::run::StepEntry`] — the **terminal summary** of one
9//! dispatched step (write-once, appended by the dispatcher after the
10//! outcome is known; carries duration / usage / model / verdict).
11//! - [`TraceEvent`] (this module) — the **in-flight stream** of what is
12//! happening (`core.step_dispatched`, `mw.long_hold_warn`, …),
13//! append-only, per-Run, ordered by `seq`.
14//!
15//! Everything here is purely observational: a failed append must never
16//! fail the dispatch it observes (callers warn-and-swallow — the same
17//! fail-open convention as `EngineDispatcher::dispatch`'s
18//! `append_step_entry`). The naming is deliberately `Trace`, not `Log`:
19//! in the Rust ecosystem "log" collides with the `log`/`tracing`
20//! facade crates, and this rail is domain data, not process logging.
21//!
22//! Kinds are an **open set** of namespaced strings — writers may insert
23//! new kinds without a schema migration. Current namespaces:
24//!
25//! - `core.*` — engine/dispatcher (`run_started` / `step_dispatched` /
26//! `step_completed` / `cancel_requested` / `run_finished`)
27//! - `mw.*` — middleware (`long_hold_warn`, …)
28//! - `worker.*` — adapter / worker self-reports
29//! - `ext.*` — future external writers (Lua flow, enhance flow, tools)
30//!
31//! Layering invariant (future `mlua-swarm-trace` crate split): this
32//! module must not depend on engine types — only `crate::types` ids and
33//! serde values.
34
35use crate::types::RunId;
36use async_trait::async_trait;
37use serde::{Deserialize, Serialize};
38use serde_json::Value;
39use std::collections::HashMap;
40use std::sync::{Arc, Mutex};
41use thiserror::Error;
42
43pub mod sqlite;
44pub use sqlite::SqliteRunTraceStore;
45
46// ──────────────────────────────────────────────────────────────────────────
47// Normalized worker stats (TokenUsage / WorkerStats)
48// ──────────────────────────────────────────────────────────────────────────
49
50/// Aggregated token usage for one worker attempt, normalized across
51/// worker kinds (agent-block `agent.run` return / subprocess declared
52/// mapping / operator self-report). Field names follow the Anthropic
53/// wire convention (`input_tokens` / `output_tokens`) that agent-block
54/// already normalizes OpenAI-style responses into.
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
56pub struct TokenUsage {
57 /// Prompt-side tokens consumed, summed across the attempt's turns.
58 pub input_tokens: u64,
59 /// Completion-side tokens produced, summed across the attempt's turns.
60 pub output_tokens: u64,
61 /// `input + output` (kept explicit because some producers report a
62 /// total that includes cache-read/creation tokens the two split
63 /// fields don't cover).
64 pub total_tokens: u64,
65}
66
67/// Normalized per-attempt worker statistics, reported by a worker
68/// boundary (spawner fold site / result captor / `POST
69/// /v1/worker/submit`) into the engine and folded into the terminal
70/// [`crate::store::run::StepEntry`] by the dispatcher.
71///
72/// The three named fields are the **closed schema** the engine knows;
73/// everything worker-kind-specific rides in [`Self::adapter_data`] as
74/// raw JSON the engine never interprets (capped at
75/// [`TRACE_PAYLOAD_CAP_BYTES`] on fold). Every field is optional —
76/// absence must never block a dispatch.
77#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
78pub struct WorkerStats {
79 /// Worker kind label (`"agent_block"` / `"subprocess"` /
80 /// `"operator"` / …) — set by whichever boundary constructed the
81 /// stats, since only that boundary knows its own kind.
82 #[serde(default, skip_serializing_if = "Option::is_none")]
83 pub worker_kind: Option<String>,
84 /// The model that served the attempt, when the boundary knows it
85 /// (subprocess: the rendered `{model}` placeholder; operator:
86 /// self-report; agent-block: spec-declared model if any).
87 #[serde(default, skip_serializing_if = "Option::is_none")]
88 pub model: Option<String>,
89 /// Normalized token usage, when the boundary can produce one.
90 #[serde(default, skip_serializing_if = "Option::is_none")]
91 pub usage: Option<TokenUsage>,
92 /// Number of LLM turns the attempt ran (agent-block `num_turns`).
93 #[serde(default, skip_serializing_if = "Option::is_none")]
94 pub num_turns: Option<u32>,
95 /// Worker-kind-specific raw payload (exit code, stderr tail, cache
96 /// token detail, …). Observational only — the engine stores it
97 /// verbatim (size-capped) and never branches on it.
98 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub adapter_data: Option<Value>,
100}
101
102impl WorkerStats {
103 /// `true` when no field carries information — callers skip
104 /// recording an all-empty stats value.
105 pub fn is_empty(&self) -> bool {
106 self.worker_kind.is_none()
107 && self.model.is_none()
108 && self.usage.is_none()
109 && self.num_turns.is_none()
110 && self.adapter_data.is_none()
111 }
112}
113
114// ──────────────────────────────────────────────────────────────────────────
115// TraceEvent / TraceQuery
116// ──────────────────────────────────────────────────────────────────────────
117
118/// Byte cap applied to [`TraceEvent::payload`] and
119/// [`WorkerStats::adapter_data`] before persisting. Oversized values are
120/// replaced by a truncation marker object (see [`cap_payload`]) — the
121/// trace rail is an observability artifact, not a blob store.
122pub const TRACE_PAYLOAD_CAP_BYTES: usize = 8 * 1024;
123
124/// Default per-Run retention ceiling — appends beyond this many events
125/// prune the oldest rows first.
126pub const DEFAULT_TRACE_MAX_EVENTS_PER_RUN: usize = 10_000;
127
128/// Default `list` page size when a query sets neither `limit` nor
129/// `latest`.
130pub const DEFAULT_TRACE_LIST_LIMIT: usize = 1_000;
131
132/// Well-known `TraceEvent.kind` values written by the engine itself.
133/// The kind axis is an open set — these constants exist so in-tree
134/// writers and tests agree on spelling, not to constrain writers.
135pub mod kind {
136 /// A Run began dispatching (server-side, once per kick).
137 pub const RUN_STARTED: &str = "core.run_started";
138 /// The dispatcher is about to spawn a step's worker.
139 pub const STEP_DISPATCHED: &str = "core.step_dispatched";
140 /// A step reached its terminal outcome (payload carries the status
141 /// label + timing summary; the authoritative record is the
142 /// `StepEntry` appended in the same breath).
143 pub const STEP_COMPLETED: &str = "core.step_completed";
144 /// A Run reached its terminal status (payload: `{"status": ...}`).
145 pub const RUN_FINISHED: &str = "core.run_finished";
146 /// Cancellation was requested for the Run.
147 pub const CANCEL_REQUESTED: &str = "core.cancel_requested";
148 /// `LongHoldMiddleware` observed a completion above its threshold.
149 pub const LONG_HOLD_WARN: &str = "mw.long_hold_warn";
150 /// A worker reported a degradation (mirrors the `DegradationEntry`
151 /// rail so the trace stream is self-contained).
152 pub const WORKER_DEGRADATION: &str = "worker.degradation";
153}
154
155/// One persisted trace event — a member of a Run's append-only stream.
156#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
157pub struct TraceEvent {
158 /// The Run this event belongs to.
159 #[schemars(with = "String")]
160 pub run_id: RunId,
161 /// Per-Run monotonically increasing ordering key, assigned by the
162 /// store at append time (1-based).
163 pub seq: u64,
164 /// Unix epoch milliseconds — when the event was recorded.
165 pub ts_ms: i64,
166 /// Namespaced kind string (open set; see the module doc).
167 pub kind: String,
168 /// The Blueprint step ref this event concerns, if any.
169 #[serde(default, skip_serializing_if = "Option::is_none")]
170 pub step_ref: Option<String>,
171 /// The attempt number this event concerns, if any.
172 #[serde(default, skip_serializing_if = "Option::is_none")]
173 pub attempt: Option<u32>,
174 /// Free-form JSON payload (capped at [`TRACE_PAYLOAD_CAP_BYTES`]).
175 pub payload: Value,
176}
177
178/// The caller-supplied half of a [`TraceEvent`] — the store assigns
179/// `seq` and `ts_ms` at append time.
180#[derive(Debug, Clone)]
181pub struct TraceEventDraft {
182 /// Namespaced kind string (open set).
183 pub kind: String,
184 /// The Blueprint step ref this event concerns, if any.
185 pub step_ref: Option<String>,
186 /// The attempt number this event concerns, if any.
187 pub attempt: Option<u32>,
188 /// Free-form JSON payload — capped by the store on append.
189 pub payload: Value,
190}
191
192/// Filter/paging parameters for [`RunTraceStore::list`]. All filters
193/// AND together; `latest` and `after` are mutually exclusive with
194/// `latest` winning (it answers "show me the tail" regardless of any
195/// cursor the caller also carried).
196#[derive(Debug, Clone, Default)]
197pub struct TraceQuery {
198 /// Forward-paging cursor: only events with `seq > after`.
199 pub after: Option<u64>,
200 /// Page size cap (defaults to [`DEFAULT_TRACE_LIST_LIMIT`]).
201 pub limit: Option<usize>,
202 /// Tail mode: return the LAST n matching events (still in ascending
203 /// `seq` order). Takes precedence over `after`/`limit`.
204 pub latest: Option<usize>,
205 /// Kind filters — an event matches when its kind equals, or starts
206 /// with, ANY entry (prefix match: `"mw."` matches every middleware
207 /// kind). Empty = no kind filter.
208 pub kinds: Vec<String>,
209 /// Exact `step_ref` filter.
210 pub step_ref: Option<String>,
211 /// Exact `attempt` filter.
212 pub attempt: Option<u32>,
213}
214
215impl TraceQuery {
216 /// Does `event` pass this query's kind/step/attempt filters
217 /// (paging axes excluded)?
218 fn matches(&self, event: &TraceEvent) -> bool {
219 if !self.kinds.is_empty()
220 && !self
221 .kinds
222 .iter()
223 .any(|k| event.kind == *k || event.kind.starts_with(k.as_str()))
224 {
225 return false;
226 }
227 if let Some(step_ref) = &self.step_ref {
228 if event.step_ref.as_deref() != Some(step_ref.as_str()) {
229 return false;
230 }
231 }
232 if let Some(attempt) = self.attempt {
233 if event.attempt != Some(attempt) {
234 return false;
235 }
236 }
237 true
238 }
239
240 /// Apply paging (latest wins over after/limit) to an ascending,
241 /// already-filtered event list.
242 fn page(&self, mut events: Vec<TraceEvent>) -> Vec<TraceEvent> {
243 if let Some(n) = self.latest {
244 let start = events.len().saturating_sub(n);
245 return events.split_off(start);
246 }
247 if let Some(after) = self.after {
248 events.retain(|e| e.seq > after);
249 }
250 let limit = self.limit.unwrap_or(DEFAULT_TRACE_LIST_LIMIT);
251 events.truncate(limit);
252 events
253 }
254}
255
256/// Replace an oversized payload with a truncation marker carrying the
257/// original size and a head excerpt, so a runaway writer cannot bloat
258/// the trace store. Values at or under [`TRACE_PAYLOAD_CAP_BYTES`] pass
259/// through unchanged.
260pub fn cap_payload(payload: Value) -> Value {
261 let serialized = payload.to_string();
262 if serialized.len() <= TRACE_PAYLOAD_CAP_BYTES {
263 return payload;
264 }
265 let head: String = serialized.chars().take(1024).collect();
266 serde_json::json!({
267 "truncated": true,
268 "size_bytes": serialized.len(),
269 "head": head,
270 })
271}
272
273/// Errors surfaced by a [`RunTraceStore`] implementation.
274#[derive(Debug, Error)]
275pub enum TraceStoreError {
276 /// Backend-specific failure.
277 #[error("other: {0}")]
278 Other(String),
279}
280
281// ──────────────────────────────────────────────────────────────────────────
282// RunTraceStore trait
283// ──────────────────────────────────────────────────────────────────────────
284
285/// Persistence interface for the per-Run trace stream.
286#[async_trait]
287pub trait RunTraceStore: Send + Sync {
288 /// Backend name — for diagnostics/logging.
289 fn name(&self) -> &str;
290
291 /// Append one event to `run_id`'s stream, assigning the next `seq`
292 /// and stamping `ts_ms`. The store caps `draft.payload` via
293 /// [`cap_payload`] and prunes the oldest rows beyond the per-Run
294 /// retention ceiling. Appending to an unknown `run_id` is legal —
295 /// the trace rail has no foreign-key coupling to `RunStore` (a
296 /// trace writer must never fail because Run-row creation raced it).
297 async fn append(
298 &self,
299 run_id: &RunId,
300 draft: TraceEventDraft,
301 ) -> Result<TraceEvent, TraceStoreError>;
302
303 /// List `run_id`'s events matching `query`, ascending by `seq`.
304 async fn list(
305 &self,
306 run_id: &RunId,
307 query: &TraceQuery,
308 ) -> Result<Vec<TraceEvent>, TraceStoreError>;
309
310 /// Delete every event belonging to `run_id`, returning the number
311 /// of deleted events. Deleting an unknown/empty Run is `Ok(0)`.
312 async fn delete_run(&self, run_id: &RunId) -> Result<u64, TraceStoreError>;
313}
314
315// ──────────────────────────────────────────────────────────────────────────
316// TraceHandle — the pervasive-insertion write port
317// ──────────────────────────────────────────────────────────────────────────
318
319/// A cheap, cloneable write handle binding one `run_id` to a
320/// [`RunTraceStore`] — the single port through which the dispatcher,
321/// middlewares (via `Engine::trace_handle`), server handlers, and any
322/// future writer append trace events. Appends are **best-effort**: a
323/// store failure is logged at `warn` and swallowed, never propagated
324/// (fail-open, matching the `append_step_entry` convention).
325#[derive(Clone)]
326pub struct TraceHandle {
327 run_id: RunId,
328 store: Arc<dyn RunTraceStore>,
329}
330
331impl TraceHandle {
332 /// Bind `run_id` to `store`.
333 pub fn new(run_id: RunId, store: Arc<dyn RunTraceStore>) -> Self {
334 Self { run_id, store }
335 }
336
337 /// The Run this handle appends into.
338 pub fn run_id(&self) -> &RunId {
339 &self.run_id
340 }
341
342 /// Best-effort append (see the struct doc). `kind` should follow
343 /// the namespaced open-set convention (`core.*` / `mw.*` /
344 /// `worker.*` / `ext.*`).
345 pub async fn append(
346 &self,
347 kind: &str,
348 step_ref: Option<&str>,
349 attempt: Option<u32>,
350 payload: Value,
351 ) {
352 let draft = TraceEventDraft {
353 kind: kind.to_string(),
354 step_ref: step_ref.map(str::to_string),
355 attempt,
356 payload,
357 };
358 if let Err(e) = self.store.append(&self.run_id, draft).await {
359 tracing::warn!(
360 run_id = %self.run_id,
361 kind = kind,
362 error = %e,
363 "TraceHandle::append failed (swallowed — trace is observational)"
364 );
365 }
366 }
367}
368
369impl std::fmt::Debug for TraceHandle {
370 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
371 f.debug_struct("TraceHandle")
372 .field("run_id", &self.run_id)
373 .field("store", &self.store.name())
374 .finish()
375 }
376}
377
378// ──────────────────────────────────────────────────────────────────────────
379// InMemoryRunTraceStore
380// ──────────────────────────────────────────────────────────────────────────
381
382#[derive(Default)]
383struct TraceInner {
384 /// Per-Run ascending event lists.
385 events: HashMap<RunId, Vec<TraceEvent>>,
386 /// Per-Run next `seq` — kept separately from `events.len()` because
387 /// retention pruning removes head entries without recycling seqs.
388 next_seq: HashMap<RunId, u64>,
389}
390
391/// Process-volatile [`RunTraceStore`] — the default when no persistent
392/// backend is wired.
393pub struct InMemoryRunTraceStore {
394 inner: Mutex<TraceInner>,
395 max_events_per_run: usize,
396}
397
398impl InMemoryRunTraceStore {
399 /// Create an empty store with the default retention ceiling.
400 pub fn new() -> Self {
401 Self {
402 inner: Mutex::new(TraceInner::default()),
403 max_events_per_run: DEFAULT_TRACE_MAX_EVENTS_PER_RUN,
404 }
405 }
406
407 /// Create an empty store with a custom per-Run retention ceiling
408 /// (tests).
409 pub fn with_max_events_per_run(max: usize) -> Self {
410 Self {
411 inner: Mutex::new(TraceInner::default()),
412 max_events_per_run: max,
413 }
414 }
415}
416
417impl Default for InMemoryRunTraceStore {
418 fn default() -> Self {
419 Self::new()
420 }
421}
422
423/// Unix epoch milliseconds now — trace events are sub-second dense, so
424/// the store stamps millis (the coarser `now_unix` seconds stay on the
425/// pre-existing `StepEntry.at` / `RunRecord` fields).
426pub(crate) fn now_unix_ms() -> i64 {
427 use std::time::{SystemTime, UNIX_EPOCH};
428 SystemTime::now()
429 .duration_since(UNIX_EPOCH)
430 .map(|d| d.as_millis() as i64)
431 .unwrap_or(0)
432}
433
434#[async_trait]
435impl RunTraceStore for InMemoryRunTraceStore {
436 fn name(&self) -> &str {
437 "in-memory"
438 }
439
440 async fn append(
441 &self,
442 run_id: &RunId,
443 draft: TraceEventDraft,
444 ) -> Result<TraceEvent, TraceStoreError> {
445 let mut inner = self.inner.lock().unwrap();
446 let seq_slot = inner.next_seq.entry(run_id.clone()).or_insert(0);
447 *seq_slot += 1;
448 let event = TraceEvent {
449 run_id: run_id.clone(),
450 seq: *seq_slot,
451 ts_ms: now_unix_ms(),
452 kind: draft.kind,
453 step_ref: draft.step_ref,
454 attempt: draft.attempt,
455 payload: cap_payload(draft.payload),
456 };
457 let list = inner.events.entry(run_id.clone()).or_default();
458 list.push(event.clone());
459 if list.len() > self.max_events_per_run {
460 let overflow = list.len() - self.max_events_per_run;
461 list.drain(..overflow);
462 }
463 Ok(event)
464 }
465
466 async fn list(
467 &self,
468 run_id: &RunId,
469 query: &TraceQuery,
470 ) -> Result<Vec<TraceEvent>, TraceStoreError> {
471 let inner = self.inner.lock().unwrap();
472 let events: Vec<TraceEvent> = inner
473 .events
474 .get(run_id)
475 .map(|list| list.iter().filter(|e| query.matches(e)).cloned().collect())
476 .unwrap_or_default();
477 Ok(query.page(events))
478 }
479
480 async fn delete_run(&self, run_id: &RunId) -> Result<u64, TraceStoreError> {
481 let mut inner = self.inner.lock().unwrap();
482 inner.next_seq.remove(run_id);
483 Ok(inner
484 .events
485 .remove(run_id)
486 .map(|list| list.len() as u64)
487 .unwrap_or(0))
488 }
489}
490
491// ──────────────────────────────────────────────────────────────────────────
492// tests
493// ──────────────────────────────────────────────────────────────────────────
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498 use serde_json::json;
499
500 fn rid(s: &str) -> RunId {
501 RunId::parse(s).unwrap()
502 }
503
504 fn draft(kind: &str, step_ref: Option<&str>, attempt: Option<u32>) -> TraceEventDraft {
505 TraceEventDraft {
506 kind: kind.to_string(),
507 step_ref: step_ref.map(str::to_string),
508 attempt,
509 payload: json!({"k": kind}),
510 }
511 }
512
513 #[tokio::test]
514 async fn append_assigns_monotonic_seq_per_run() {
515 let s = InMemoryRunTraceStore::new();
516 let e1 = s
517 .append(&rid("R-1"), draft("core.run_started", None, None))
518 .await
519 .unwrap();
520 let e2 = s
521 .append(
522 &rid("R-1"),
523 draft("core.step_dispatched", Some("w"), Some(1)),
524 )
525 .await
526 .unwrap();
527 let other = s
528 .append(&rid("R-2"), draft("core.run_started", None, None))
529 .await
530 .unwrap();
531 assert_eq!(e1.seq, 1);
532 assert_eq!(e2.seq, 2);
533 assert_eq!(other.seq, 1, "seq is per-Run, not global");
534 assert!(e1.ts_ms > 0);
535 }
536
537 #[tokio::test]
538 async fn list_filters_by_kind_prefix_step_and_attempt() {
539 let s = InMemoryRunTraceStore::new();
540 let r = rid("R-1");
541 s.append(&r, draft("core.run_started", None, None))
542 .await
543 .unwrap();
544 s.append(&r, draft("core.step_dispatched", Some("a"), Some(1)))
545 .await
546 .unwrap();
547 s.append(&r, draft("mw.long_hold_warn", Some("a"), Some(1)))
548 .await
549 .unwrap();
550 s.append(&r, draft("core.step_completed", Some("b"), Some(2)))
551 .await
552 .unwrap();
553
554 let mw = s
555 .list(
556 &r,
557 &TraceQuery {
558 kinds: vec!["mw.".into()],
559 ..Default::default()
560 },
561 )
562 .await
563 .unwrap();
564 assert_eq!(mw.len(), 1);
565 assert_eq!(mw[0].kind, "mw.long_hold_warn");
566
567 let step_a = s
568 .list(
569 &r,
570 &TraceQuery {
571 step_ref: Some("a".into()),
572 ..Default::default()
573 },
574 )
575 .await
576 .unwrap();
577 assert_eq!(step_a.len(), 2);
578
579 let attempt2 = s
580 .list(
581 &r,
582 &TraceQuery {
583 attempt: Some(2),
584 ..Default::default()
585 },
586 )
587 .await
588 .unwrap();
589 assert_eq!(attempt2.len(), 1);
590 assert_eq!(attempt2[0].step_ref.as_deref(), Some("b"));
591 }
592
593 #[tokio::test]
594 async fn list_paging_after_and_latest() {
595 let s = InMemoryRunTraceStore::new();
596 let r = rid("R-1");
597 for i in 0..5 {
598 s.append(&r, draft(&format!("core.e{i}"), None, None))
599 .await
600 .unwrap();
601 }
602
603 let after = s
604 .list(
605 &r,
606 &TraceQuery {
607 after: Some(3),
608 ..Default::default()
609 },
610 )
611 .await
612 .unwrap();
613 assert_eq!(after.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![4, 5]);
614
615 let latest = s
616 .list(
617 &r,
618 &TraceQuery {
619 latest: Some(2),
620 // latest must win over a cursor the caller also set.
621 after: Some(1),
622 ..Default::default()
623 },
624 )
625 .await
626 .unwrap();
627 assert_eq!(latest.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![4, 5]);
628
629 let limited = s
630 .list(
631 &r,
632 &TraceQuery {
633 limit: Some(2),
634 ..Default::default()
635 },
636 )
637 .await
638 .unwrap();
639 assert_eq!(
640 limited.iter().map(|e| e.seq).collect::<Vec<_>>(),
641 vec![1, 2]
642 );
643 }
644
645 #[tokio::test]
646 async fn retention_prunes_oldest_keeping_seq() {
647 let s = InMemoryRunTraceStore::with_max_events_per_run(3);
648 let r = rid("R-1");
649 for i in 0..5 {
650 s.append(&r, draft(&format!("core.e{i}"), None, None))
651 .await
652 .unwrap();
653 }
654 let all = s.list(&r, &TraceQuery::default()).await.unwrap();
655 assert_eq!(all.iter().map(|e| e.seq).collect::<Vec<_>>(), vec![3, 4, 5]);
656 // A later append keeps counting up — pruning never recycles seqs.
657 let e6 = s.append(&r, draft("core.e5", None, None)).await.unwrap();
658 assert_eq!(e6.seq, 6);
659 }
660
661 #[tokio::test]
662 async fn delete_run_removes_stream() {
663 let s = InMemoryRunTraceStore::new();
664 let r = rid("R-1");
665 s.append(&r, draft("core.run_started", None, None))
666 .await
667 .unwrap();
668 s.append(&r, draft("core.run_finished", None, None))
669 .await
670 .unwrap();
671 assert_eq!(s.delete_run(&r).await.unwrap(), 2);
672 assert!(s.list(&r, &TraceQuery::default()).await.unwrap().is_empty());
673 assert_eq!(s.delete_run(&r).await.unwrap(), 0, "double delete is Ok(0)");
674 }
675
676 #[tokio::test]
677 async fn oversized_payload_is_truncated_with_marker() {
678 let s = InMemoryRunTraceStore::new();
679 let r = rid("R-1");
680 let big = "x".repeat(TRACE_PAYLOAD_CAP_BYTES + 100);
681 let e = s
682 .append(
683 &r,
684 TraceEventDraft {
685 kind: "worker.output".into(),
686 step_ref: None,
687 attempt: None,
688 payload: json!({"blob": big}),
689 },
690 )
691 .await
692 .unwrap();
693 assert_eq!(e.payload.get("truncated"), Some(&json!(true)));
694 assert!(e.payload.get("size_bytes").is_some());
695 }
696
697 #[tokio::test]
698 async fn trace_handle_appends_best_effort() {
699 let store: Arc<dyn RunTraceStore> = Arc::new(InMemoryRunTraceStore::new());
700 let handle = TraceHandle::new(rid("R-1"), store.clone());
701 handle
702 .append(kind::STEP_DISPATCHED, Some("w"), Some(1), json!({}))
703 .await;
704 let events = store
705 .list(&rid("R-1"), &TraceQuery::default())
706 .await
707 .unwrap();
708 assert_eq!(events.len(), 1);
709 assert_eq!(events[0].kind, kind::STEP_DISPATCHED);
710 }
711
712 #[test]
713 fn worker_stats_is_empty_reflects_fields() {
714 assert!(WorkerStats::default().is_empty());
715 let stats = WorkerStats {
716 usage: Some(TokenUsage {
717 input_tokens: 1,
718 output_tokens: 2,
719 total_tokens: 3,
720 }),
721 ..Default::default()
722 };
723 assert!(!stats.is_empty());
724 }
725}