lash_trace/
lib.rs

1//! Durable diagnostics for the lash runtime: the [`TraceSink`] channel and its
2//! record vocabulary.
3//!
4//! A [`TraceSink`] receives one [`TraceRecord`] per runtime event — session and
5//! turn lifecycle, prompt builds, LLM calls, per-tool start/completion, token
6//! usage, protocol steps, and Lashlang execution-graph updates. Each record
7//! carries a [`TraceContext`] (session / turn / graph-node identity) plus a
8//! tagged [`TraceEvent`] payload; [`TraceEvent::kind`] is the single source of
9//! truth for the `type` tag string that consumers match on.
10//!
11//! [`JsonlTraceSink`] writes one JSON line per record at schema
12//! [`TRACE_SCHEMA_VERSION`]; [`TeeTraceSink`] fans out to several sinks; and the
13//! optional `otel` feature adds an `OtelTraceSink` that converts each record to
14//! an OpenTelemetry span. This is the *durable diagnostics* reporting channel —
15//! distinct from the app-facing `TurnActivity` stream and the low-level
16//! `SessionStreamEvent` stream that the runtime crates expose.
17//!
18//! For the full map of reporting channels, guidance on when to consume which,
19//! and the schema-evolution policy that governs [`TRACE_SCHEMA_VERSION`], see
20//! `docs/reporting.html`; for the attach-a-sink how-to, see `docs/tracing.html`.
21
22use std::collections::BTreeMap;
23use std::fs::OpenOptions;
24use std::io::{self, Write};
25use std::path::{Path, PathBuf};
26use std::sync::{Arc, Mutex};
27
28use serde::{Deserialize, Serialize};
29use serde_json::Value;
30use sha2::{Digest, Sha256};
31
32mod lashlang_graph;
33#[cfg(feature = "otel")]
34pub mod otel;
35
36pub use lashlang_graph::{
37    TraceLashlangEdgeSelection, TraceLashlangGraph, TraceLashlangGraphChildLink,
38    TraceLashlangGraphEdge, TraceLashlangGraphNode, TraceLashlangGraphStore,
39    TraceLashlangNodeStatus,
40};
41
42/// Version of the durable trace JSONL schema, written to
43/// [`TraceRecord::schema_version`] on every record.
44///
45/// Bump rules (the normative reporting-schema policy lives in
46/// `docs/reporting.html`):
47///
48/// - Adding a new [`TraceEvent`] variant, or adding an optional
49///   (`skip_serializing_if`) field to an existing payload, is **additive**:
50///   older readers skip the unknown variant or field, so this version does
51///   **not** change.
52/// - Renaming a field, removing a field, or changing the meaning of an existing
53///   field is a breaking change and **does** bump this version.
54/// - The free-form [`TraceEvent::Custom`] and [`TraceEvent::ProtocolStep`]
55///   payloads are opaque `serde_json::Value`; adding to or reshaping the data
56///   inside them never forces a bump. (This is why the `exec_code_completed`
57///   diagnostic's `tool_calls` payload was purely additive.)
58pub const TRACE_SCHEMA_VERSION: u32 = 2;
59
60#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
61#[serde(rename_all = "snake_case")]
62pub enum TraceLevel {
63    #[default]
64    Standard,
65    Extended,
66}
67
68impl TraceLevel {
69    pub fn is_extended(self) -> bool {
70        matches!(self, Self::Extended)
71    }
72}
73
74#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
75pub struct TraceContext {
76    #[serde(default, skip_serializing_if = "Option::is_none")]
77    pub run_id: Option<String>,
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub experiment_id: Option<String>,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub candidate_id: Option<String>,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub candidate_parent_id: Option<String>,
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub example_id: Option<String>,
86    #[serde(default, skip_serializing_if = "Option::is_none")]
87    pub split: Option<String>,
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub session_id: Option<String>,
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub turn_id: Option<String>,
92    /// Stable id of the span this record represents (e.g. `turn:<session>:<turn>`,
93    /// `llm:<call_id>`, `tool:<call_id>`). Populated by the runtime for turn /
94    /// llm / tool / session records so a consumer can build a nested span tree
95    /// from `(graph_node_id, parent_graph_node_id)` with a single `id -> span`
96    /// map. Lashlang execution sets its own graph node id here.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub graph_node_id: Option<String>,
99    /// Id of the enclosing span — the value of some other record's
100    /// `graph_node_id`. A turn's parent is its causal origin (the spawning tool
101    /// call / effect, via `caused_by`) when known, otherwise the session root.
102    #[serde(default, skip_serializing_if = "Option::is_none")]
103    pub parent_graph_node_id: Option<String>,
104    #[serde(default, skip_serializing_if = "Option::is_none")]
105    pub turn_index: Option<usize>,
106    #[serde(default, skip_serializing_if = "Option::is_none")]
107    pub protocol_iteration: Option<usize>,
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub effect_id: Option<String>,
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub llm_call_id: Option<String>,
112    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
113    pub metadata: BTreeMap<String, Value>,
114}
115
116impl TraceContext {
117    pub fn for_session(mut self, session_id: impl Into<String>) -> Self {
118        self.session_id = Some(session_id.into());
119        self
120    }
121
122    pub fn for_turn_index(mut self, turn_index: usize) -> Self {
123        self.turn_index = Some(turn_index);
124        self
125    }
126
127    pub fn for_turn(mut self, turn_id: impl Into<String>) -> Self {
128        self.turn_id = Some(turn_id.into());
129        self
130    }
131
132    pub fn for_protocol_iteration(mut self, protocol_iteration: usize) -> Self {
133        self.protocol_iteration = Some(protocol_iteration);
134        self
135    }
136
137    pub fn for_llm_call(mut self, llm_call_id: impl Into<String>) -> Self {
138        self.llm_call_id = Some(llm_call_id.into());
139        self
140    }
141}
142
143#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
144pub struct TraceRecord {
145    pub schema_version: u32,
146    pub id: String,
147    pub timestamp: String,
148    pub context: TraceContext,
149    #[serde(flatten)]
150    pub event: TraceEvent,
151}
152
153impl TraceRecord {
154    pub fn new(context: TraceContext, event: TraceEvent) -> Self {
155        Self::new_with_timestamp(context, event, chrono::Utc::now())
156    }
157
158    pub fn new_with_timestamp(
159        context: TraceContext,
160        event: TraceEvent,
161        timestamp: chrono::DateTime<chrono::Utc>,
162    ) -> Self {
163        Self {
164            schema_version: TRACE_SCHEMA_VERSION,
165            id: uuid::Uuid::new_v4().to_string(),
166            timestamp: timestamp.to_rfc3339(),
167            context,
168            event,
169        }
170    }
171}
172
173#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
174#[serde(tag = "type", rename_all = "snake_case")]
175#[allow(
176    clippy::large_enum_variant,
177    reason = "TraceEvent is a public DTO; keeping event payloads inline preserves ergonomic pattern matching"
178)]
179pub enum TraceEvent {
180    SessionStarted {
181        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
182        metadata: BTreeMap<String, Value>,
183    },
184    TurnStarted {
185        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
186        metadata: BTreeMap<String, Value>,
187    },
188    PromptBuilt {
189        prompt_hash: String,
190        prompt_chars: usize,
191        #[serde(default, skip_serializing_if = "Vec::is_empty")]
192        components: Vec<TracePromptComponent>,
193    },
194    LlmCallStarted {
195        request: TraceLlmRequest,
196    },
197    LlmCallCompleted {
198        response: TraceLlmResponse,
199        #[serde(default, skip_serializing_if = "Option::is_none")]
200        usage: Option<TraceTokenUsage>,
201        #[serde(default, skip_serializing_if = "Option::is_none")]
202        provider_usage: Option<Value>,
203        #[serde(default, skip_serializing_if = "Option::is_none")]
204        stream_summary: Option<Value>,
205    },
206    LlmCallFailed {
207        error: TraceError,
208        #[serde(default, skip_serializing_if = "Option::is_none")]
209        stream_summary: Option<Value>,
210    },
211    ProviderRequest {
212        event: TraceProviderRequestEvent,
213    },
214    EffectEnvelopeDiff {
215        event: TraceEffectEnvelopeDiffEvent,
216    },
217    ProviderStreamEvent {
218        event: TraceProviderStreamEvent,
219    },
220    RuntimeStreamEvent {
221        event: TraceRuntimeStreamEvent,
222    },
223    ToolCallStarted {
224        call_id: Option<String>,
225        name: String,
226        args: Value,
227    },
228    ToolCallCompleted {
229        call_id: Option<String>,
230        name: String,
231        args: Value,
232        output: TraceToolCallOutput,
233        duration_ms: u64,
234    },
235    ProtocolStep {
236        plugin_id: String,
237        payload: Value,
238    },
239    TokenUsage {
240        usage: TraceTokenUsage,
241        #[serde(default, skip_serializing_if = "Option::is_none")]
242        cumulative: Option<TraceTokenUsage>,
243    },
244    LashlangExecution {
245        event: TraceLashlangExecutionEvent,
246    },
247    TurnCompleted {
248        status: String,
249        done_reason: String,
250        #[serde(default, skip_serializing_if = "Option::is_none")]
251        agent_frame_switch: Option<TraceAgentFrameSwitch>,
252    },
253    Custom {
254        name: String,
255        payload: Value,
256    },
257}
258
259impl TraceEvent {
260    /// The `type` tag serde writes for this variant. This is the single source
261    /// of truth for event-kind strings — consumers (e.g. the trace viewer)
262    /// match on the enum and read the kind from here rather than re-deriving
263    /// tag strings by hand. The match is exhaustive on purpose: a new variant
264    /// fails to compile here until it is given a kind.
265    pub fn kind(&self) -> &'static str {
266        match self {
267            Self::SessionStarted { .. } => "session_started",
268            Self::TurnStarted { .. } => "turn_started",
269            Self::PromptBuilt { .. } => "prompt_built",
270            Self::LlmCallStarted { .. } => "llm_call_started",
271            Self::LlmCallCompleted { .. } => "llm_call_completed",
272            Self::LlmCallFailed { .. } => "llm_call_failed",
273            Self::ProviderRequest { .. } => "provider_request",
274            Self::EffectEnvelopeDiff { .. } => "effect_envelope_diff",
275            Self::ProviderStreamEvent { .. } => "provider_stream_event",
276            Self::RuntimeStreamEvent { .. } => "runtime_stream_event",
277            Self::ToolCallStarted { .. } => "tool_call_started",
278            Self::ToolCallCompleted { .. } => "tool_call_completed",
279            Self::ProtocolStep { .. } => "protocol_step",
280            Self::TokenUsage { .. } => "token_usage",
281            Self::LashlangExecution { .. } => "lashlang_execution",
282            Self::TurnCompleted { .. } => "turn_completed",
283            Self::Custom { .. } => "custom",
284        }
285    }
286}
287
288#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
289pub struct TraceToolCallOutput {
290    pub outcome: TraceToolCallOutcome,
291    #[serde(default, skip_serializing_if = "Option::is_none")]
292    pub control: Option<Value>,
293}
294
295impl TraceToolCallOutput {
296    pub fn status(&self) -> TraceToolCallStatus {
297        match self.outcome {
298            TraceToolCallOutcome::Success(_) => TraceToolCallStatus::Success,
299            TraceToolCallOutcome::Failure(_) => TraceToolCallStatus::Failure,
300            TraceToolCallOutcome::Cancelled(_) => TraceToolCallStatus::Cancelled,
301        }
302    }
303
304    pub fn is_success(&self) -> bool {
305        self.status() == TraceToolCallStatus::Success
306    }
307
308    pub fn value_for_projection(&self) -> Value {
309        match &self.outcome {
310            TraceToolCallOutcome::Success(value)
311            | TraceToolCallOutcome::Failure(value)
312            | TraceToolCallOutcome::Cancelled(value) => value.clone(),
313        }
314    }
315}
316
317#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
318#[serde(tag = "status", content = "payload", rename_all = "snake_case")]
319pub enum TraceToolCallOutcome {
320    Success(Value),
321    Failure(Value),
322    Cancelled(Value),
323}
324
325#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
326#[serde(rename_all = "snake_case")]
327pub enum TraceToolCallStatus {
328    Success,
329    Failure,
330    Cancelled,
331}
332
333#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
334pub struct TracePromptComponent {
335    pub id: String,
336    pub kind: String,
337    pub hash: String,
338    #[serde(default, skip_serializing_if = "Option::is_none")]
339    pub chars: Option<usize>,
340}
341
342#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
343pub struct TraceLlmRequest {
344    pub model: String,
345    #[serde(default, skip_serializing_if = "Option::is_none")]
346    pub model_variant: Option<String>,
347    pub messages: Vec<TraceLlmMessage>,
348    #[serde(default, skip_serializing_if = "Vec::is_empty")]
349    pub attachments: Vec<TraceAttachment>,
350    #[serde(default, skip_serializing_if = "Vec::is_empty")]
351    pub tools: Vec<TraceToolSpec>,
352    pub tool_choice: String,
353    #[serde(default, skip_serializing_if = "Option::is_none")]
354    pub output_spec: Option<Value>,
355    pub stream: bool,
356}
357
358#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
359pub struct TraceLlmMessage {
360    pub role: String,
361    pub blocks: Vec<TraceContentBlock>,
362}
363
364#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
365#[serde(tag = "kind", rename_all = "snake_case")]
366pub enum TraceContentBlock {
367    Text {
368        text: String,
369        #[serde(default, skip_serializing_if = "is_false")]
370        cache_breakpoint: bool,
371    },
372    Attachment {
373        attachment_idx: usize,
374    },
375    ToolCall {
376        call_id: Option<String>,
377        tool_name: String,
378        input_json: Value,
379        item_id: Option<String>,
380        has_signature: bool,
381    },
382    ToolResult {
383        call_id: Option<String>,
384        tool_name: Option<String>,
385        content: String,
386    },
387    Reasoning {
388        text: String,
389        item_id: Option<String>,
390        summary: Vec<String>,
391        has_encrypted: bool,
392        redacted: bool,
393    },
394}
395
396fn is_false(value: &bool) -> bool {
397    !*value
398}
399
400#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
401pub struct TraceAttachment {
402    pub source: String,
403    #[serde(default, skip_serializing_if = "Option::is_none")]
404    pub mime: Option<String>,
405    #[serde(default, skip_serializing_if = "Option::is_none")]
406    pub filename: Option<String>,
407    #[serde(default, skip_serializing_if = "Option::is_none")]
408    pub bytes_sha256: Option<String>,
409    #[serde(default, skip_serializing_if = "Option::is_none")]
410    pub bytes_len: Option<usize>,
411}
412
413#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
414pub struct TraceToolSpec {
415    pub name: String,
416    pub description: String,
417    pub input_schema: Value,
418    pub output_schema: Value,
419}
420
421#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
422pub struct TraceLlmResponse {
423    pub text: String,
424    pub duration_ms: u64,
425    #[serde(default, skip_serializing_if = "Option::is_none")]
426    pub terminal_reason: Option<String>,
427    #[serde(default, skip_serializing_if = "Option::is_none")]
428    pub parts: Option<Value>,
429}
430
431#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
432pub struct TraceProviderRequestEvent {
433    pub provider: String,
434    pub sequence: u64,
435    pub elapsed_ms: u64,
436    pub endpoint: String,
437    pub body_len: usize,
438    /// SHA-256 of the exact serialized wire bytes. `body_json` is a parsed
439    /// structured view whose re-serialization may not reproduce these bytes.
440    pub body_sha256: String,
441    #[serde(default, skip_serializing_if = "Option::is_none")]
442    pub body_json: Option<Value>,
443    /// Why `body_json` is absent when the request body itself was observed.
444    #[serde(default, skip_serializing_if = "Option::is_none")]
445    pub body_json_omitted_reason: Option<String>,
446}
447
448/// Structural differences between the canonical envelopes on a failed durable
449/// replay validation. This event may contain prompt and tool-result values and
450/// must therefore only be emitted through the extended-trace gate.
451#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
452pub struct TraceEffectEnvelopeDiffEvent {
453    pub recorded_envelope_hash: String,
454    pub reconstructed_envelope_hash: String,
455    pub divergent_paths: Vec<TraceEffectEnvelopeDiffEntry>,
456}
457
458#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
459pub struct TraceEffectEnvelopeDiffEntry {
460    pub path: String,
461    pub recorded: TraceEffectEnvelopeDiffValue,
462    pub reconstructed: TraceEffectEnvelopeDiffValue,
463}
464
465/// One side of a divergent canonical-envelope value.
466///
467/// Large values are omitted whole. Their exact serialized length and digest
468/// remain available, but no prefix is retained.
469#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
470#[serde(tag = "state", rename_all = "snake_case")]
471pub enum TraceEffectEnvelopeDiffValue {
472    Missing,
473    Present {
474        json_len: usize,
475        json_sha256: String,
476        #[serde(default, skip_serializing_if = "Option::is_none")]
477        value_json: Option<Value>,
478        #[serde(default, skip_serializing_if = "Option::is_none")]
479        value_json_omitted_reason: Option<String>,
480    },
481}
482
483#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
484pub struct TraceProviderStreamEvent {
485    pub provider: String,
486    pub sequence: u64,
487    pub elapsed_ms: u64,
488    pub event_name: String,
489    #[serde(default, skip_serializing_if = "Option::is_none")]
490    pub item_id: Option<String>,
491    #[serde(default, skip_serializing_if = "Option::is_none")]
492    pub output_index: Option<i64>,
493    pub raw_len: usize,
494    pub raw_sha256: String,
495    #[serde(default, skip_serializing_if = "Option::is_none")]
496    pub raw_json: Option<Value>,
497}
498
499#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
500pub struct TraceRuntimeStreamEvent {
501    pub sequence: u64,
502    pub elapsed_ms: u64,
503    pub event_name: String,
504    #[serde(default, skip_serializing_if = "Option::is_none")]
505    pub raw_text: Option<String>,
506    #[serde(default, skip_serializing_if = "Option::is_none")]
507    pub visible_text: Option<String>,
508    #[serde(default, skip_serializing_if = "Option::is_none")]
509    pub item_id: Option<String>,
510    #[serde(default, skip_serializing_if = "Option::is_none")]
511    pub output_index: Option<i64>,
512    #[serde(default, skip_serializing_if = "Option::is_none")]
513    pub call_id: Option<String>,
514    #[serde(default, skip_serializing_if = "Option::is_none")]
515    pub tool_name: Option<String>,
516    #[serde(default, skip_serializing_if = "Option::is_none")]
517    pub input_json: Option<Value>,
518    #[serde(default, skip_serializing_if = "Option::is_none")]
519    pub usage: Option<TraceTokenUsage>,
520}
521
522#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
523pub struct TraceTokenUsage {
524    pub input_tokens: i64,
525    pub output_tokens: i64,
526    pub cache_read_input_tokens: i64,
527    pub cache_write_input_tokens: i64,
528    pub reasoning_output_tokens: i64,
529}
530
531#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
532pub struct TraceAgentFrameSwitch {
533    pub frame_id: String,
534}
535
536#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
537pub struct TraceRuntimeScope {
538    pub session_id: String,
539    #[serde(default, skip_serializing_if = "Option::is_none")]
540    pub turn_id: Option<String>,
541    #[serde(default, skip_serializing_if = "Option::is_none")]
542    pub turn_index: Option<usize>,
543    #[serde(default, skip_serializing_if = "Option::is_none")]
544    pub protocol_iteration: Option<usize>,
545}
546
547impl TraceRuntimeScope {
548    pub fn new(session_id: impl Into<String>) -> Self {
549        Self {
550            session_id: session_id.into(),
551            turn_id: None,
552            turn_index: None,
553            protocol_iteration: None,
554        }
555    }
556}
557
558#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
559#[serde(tag = "type", rename_all = "snake_case")]
560pub enum TraceRuntimeSubject {
561    Effect { effect_id: String, kind: String },
562    Process { process_id: String },
563}
564
565impl TraceRuntimeSubject {
566    pub fn graph_key(&self, scope: &TraceRuntimeScope) -> String {
567        match self {
568            Self::Effect { effect_id, .. } => match scope.turn_id.as_deref() {
569                Some(turn_id) if !turn_id.is_empty() => {
570                    format!("effect:{}:{turn_id}:{effect_id}", scope.session_id)
571                }
572                _ => format!("effect:{}:{effect_id}", scope.session_id),
573            },
574            Self::Process { process_id } => format!("process:{process_id}"),
575        }
576    }
577}
578
579#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
580pub struct TraceLashlangExecutionIdentity {
581    pub scope: TraceRuntimeScope,
582    pub subject: TraceRuntimeSubject,
583    pub module_ref: String,
584    pub entry_kind: String,
585    #[serde(default, skip_serializing_if = "Option::is_none")]
586    pub entry_ref: Option<String>,
587    pub entry_name: String,
588}
589
590impl TraceLashlangExecutionIdentity {
591    pub fn graph_key(&self) -> String {
592        self.subject.graph_key(&self.scope)
593    }
594}
595
596#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
597#[serde(tag = "kind", rename_all = "snake_case")]
598pub enum TraceLashlangExecutionEvent {
599    ExecutionStarted {
600        event_key: String,
601        identity: TraceLashlangExecutionIdentity,
602        execution_map: TraceLashlangMap,
603    },
604    ExecutionFinished {
605        event_key: String,
606        identity: TraceLashlangExecutionIdentity,
607        status: TraceLashlangStatus,
608        #[serde(default, skip_serializing_if = "Option::is_none")]
609        error: Option<String>,
610    },
611    NodeStarted {
612        event_key: String,
613        identity: TraceLashlangExecutionIdentity,
614        node_id: String,
615        node_kind: String,
616        label: String,
617        occurrence: u64,
618    },
619    NodeCompleted {
620        event_key: String,
621        identity: TraceLashlangExecutionIdentity,
622        node_id: String,
623        node_kind: String,
624        label: String,
625        occurrence: u64,
626    },
627    NodeFailed {
628        event_key: String,
629        identity: TraceLashlangExecutionIdentity,
630        node_id: String,
631        node_kind: String,
632        label: String,
633        occurrence: u64,
634        error: String,
635    },
636    BranchSelected {
637        event_key: String,
638        identity: TraceLashlangExecutionIdentity,
639        node_id: String,
640        occurrence: u64,
641        edge_id: String,
642        selected: TraceBranchSelection,
643    },
644    ChildStarted {
645        event_key: String,
646        identity: TraceLashlangExecutionIdentity,
647        parent_node_id: String,
648        occurrence: u64,
649        child: TraceLashlangChildExecution,
650    },
651}
652
653#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
654pub struct TraceLashlangChildExecution {
655    pub scope: TraceRuntimeScope,
656    pub subject: TraceRuntimeSubject,
657    #[serde(default, skip_serializing_if = "Option::is_none")]
658    pub module_ref: Option<String>,
659    #[serde(default, skip_serializing_if = "Option::is_none")]
660    pub entry_ref: Option<String>,
661    #[serde(default, skip_serializing_if = "Option::is_none")]
662    pub entry_name: Option<String>,
663}
664
665impl TraceLashlangChildExecution {
666    pub fn graph_key(&self) -> String {
667        self.subject.graph_key(&self.scope)
668    }
669}
670
671#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
672#[serde(rename_all = "snake_case")]
673pub enum TraceLashlangStatus {
674    Running,
675    Completed,
676    Failed,
677    Cancelled,
678}
679
680#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
681#[serde(rename_all = "snake_case")]
682pub enum TraceBranchSelection {
683    Then,
684    Else,
685}
686
687#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
688pub struct TraceLashlangMap {
689    pub module_ref: String,
690    pub entry_kind: String,
691    #[serde(default, skip_serializing_if = "Option::is_none")]
692    pub entry_ref: Option<String>,
693    pub entry_name: String,
694    #[serde(default)]
695    pub nodes: Vec<TraceLashlangMapNode>,
696    #[serde(default)]
697    pub edges: Vec<TraceLashlangMapEdge>,
698}
699
700#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
701pub struct TraceLashlangMapNode {
702    pub id: String,
703    pub kind: String,
704    pub label: String,
705    #[serde(default, skip_serializing_if = "Option::is_none")]
706    pub label_metadata: Option<TraceLabelMetadata>,
707}
708
709#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
710pub struct TraceLabelMetadata {
711    pub title: String,
712    #[serde(default, skip_serializing_if = "Option::is_none")]
713    pub description: Option<String>,
714}
715
716#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
717pub struct TraceLashlangMapEdge {
718    pub id: String,
719    pub from: String,
720    pub to: String,
721    pub label: String,
722}
723
724#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
725pub struct TraceError {
726    pub message: String,
727    pub retryable: bool,
728    #[serde(default, skip_serializing_if = "Option::is_none")]
729    pub terminal_reason: Option<String>,
730    #[serde(default, skip_serializing_if = "Option::is_none")]
731    pub code: Option<String>,
732    #[serde(default, skip_serializing_if = "Option::is_none")]
733    pub raw: Option<String>,
734}
735
736#[derive(Debug, thiserror::Error)]
737pub enum TraceSinkError {
738    #[error("failed to serialize trace record: {0}")]
739    Serialize(#[from] serde_json::Error),
740    #[error("trace sink lock poisoned")]
741    LockPoisoned,
742    #[error("failed to create trace directory {path}: {source}")]
743    CreateDir { path: PathBuf, source: io::Error },
744    #[error("failed to open trace file {path}: {source}")]
745    Open { path: PathBuf, source: io::Error },
746    #[error("failed to write trace file {path}: {source}")]
747    Write { path: PathBuf, source: io::Error },
748}
749
750pub trait TraceSink: Send + Sync {
751    fn append(&self, record: &TraceRecord) -> Result<(), TraceSinkError>;
752
753    /// Force any buffered trace data this sink owns to durable storage.
754    ///
755    /// Hosts call this before process exit so records that a sink has not yet
756    /// committed are not lost. The default is a no-op: sinks that write each
757    /// record through on [`append`](Self::append) (or that delegate durability
758    /// to a host-owned exporter) have nothing of their own to flush. Sinks that
759    /// buffer — or that can force an `fsync` — override this.
760    fn flush(&self) -> Result<(), TraceSinkError> {
761        Ok(())
762    }
763}
764
765pub struct JsonlTraceSink {
766    path: PathBuf,
767    lock: Mutex<()>,
768}
769
770impl JsonlTraceSink {
771    pub fn new(path: impl Into<PathBuf>) -> Self {
772        Self {
773            path: path.into(),
774            lock: Mutex::new(()),
775        }
776    }
777
778    pub fn path(&self) -> &Path {
779        &self.path
780    }
781}
782
783impl TraceSink for JsonlTraceSink {
784    fn append(&self, record: &TraceRecord) -> Result<(), TraceSinkError> {
785        let line = serde_json::to_string(record)?;
786        let _guard = self.lock.lock().map_err(|_| TraceSinkError::LockPoisoned)?;
787        if let Some(parent) = self.path.parent()
788            && !parent.as_os_str().is_empty()
789        {
790            std::fs::create_dir_all(parent).map_err(|source| TraceSinkError::CreateDir {
791                path: parent.to_path_buf(),
792                source,
793            })?;
794        }
795        let mut file = OpenOptions::new()
796            .create(true)
797            .append(true)
798            .open(&self.path)
799            .map_err(|source| TraceSinkError::Open {
800                path: self.path.clone(),
801                source,
802            })?;
803        writeln!(file, "{line}").map_err(|source| TraceSinkError::Write {
804            path: self.path.clone(),
805            source,
806        })
807    }
808
809    /// `fsync` the trace file to durable storage.
810    ///
811    /// Each [`append`](Self::append) opens, appends, and closes the file, so a
812    /// record's bytes already reach the OS as it is written — this sink holds no
813    /// in-process buffer. `flush` goes one step further and forces an `fsync` so
814    /// the OS page cache is pushed to disk, which is the honest guarantee a host
815    /// wants before exit. If no record has been written yet the file may not
816    /// exist; that is a no-op rather than an error (nothing to sync), and we do
817    /// not create an empty file just to sync it.
818    fn flush(&self) -> Result<(), TraceSinkError> {
819        let _guard = self.lock.lock().map_err(|_| TraceSinkError::LockPoisoned)?;
820        match OpenOptions::new().append(true).open(&self.path) {
821            Ok(file) => file.sync_all().map_err(|source| TraceSinkError::Write {
822                path: self.path.clone(),
823                source,
824            }),
825            Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
826            Err(source) => Err(TraceSinkError::Open {
827                path: self.path.clone(),
828                source,
829            }),
830        }
831    }
832}
833
834/// Writes each trace record as one JSON line to stderr — handy for `cargo run`
835/// debugging without a trace file.
836#[derive(Default)]
837pub struct StderrTraceSink {
838    lock: Mutex<()>,
839}
840
841impl TraceSink for StderrTraceSink {
842    fn append(&self, record: &TraceRecord) -> Result<(), TraceSinkError> {
843        let line = serde_json::to_string(record)?;
844        let _guard = self.lock.lock().map_err(|_| TraceSinkError::LockPoisoned)?;
845        eprintln!("{line}");
846        Ok(())
847    }
848}
849
850/// Fans each trace record out to several sinks in order (e.g. stderr + a JSONL
851/// file). Stops at the first sink that errors.
852pub struct TeeTraceSink {
853    sinks: Vec<Arc<dyn TraceSink>>,
854}
855
856impl TeeTraceSink {
857    pub fn new(sinks: impl IntoIterator<Item = Arc<dyn TraceSink>>) -> Self {
858        Self {
859            sinks: sinks.into_iter().collect(),
860        }
861    }
862}
863
864impl TraceSink for TeeTraceSink {
865    fn append(&self, record: &TraceRecord) -> Result<(), TraceSinkError> {
866        for sink in &self.sinks {
867            sink.append(record)?;
868        }
869        Ok(())
870    }
871
872    /// Flush every wrapped sink, stopping at the first that errors.
873    fn flush(&self) -> Result<(), TraceSinkError> {
874        for sink in &self.sinks {
875            sink.flush()?;
876        }
877        Ok(())
878    }
879}
880
881pub fn sha256_hex(input: impl AsRef<[u8]>) -> String {
882    let mut hasher = Sha256::new();
883    hasher.update(input.as_ref());
884    format!("{:x}", hasher.finalize())
885}
886
887pub fn json_hash(value: &Value) -> String {
888    sha256_hex(serde_json::to_vec(value).unwrap_or_default())
889}
890
891#[cfg(test)]
892mod tests {
893    use super::*;
894
895    #[test]
896    fn jsonl_sink_writes_record() {
897        let dir = std::env::temp_dir().join(format!("lash-trace-{}", uuid::Uuid::new_v4()));
898        std::fs::create_dir_all(&dir).unwrap();
899        let path = dir.join("trace.jsonl");
900        let sink = JsonlTraceSink::new(&path);
901        sink.append(&TraceRecord::new(
902            TraceContext::default().for_session("root"),
903            TraceEvent::Custom {
904                name: "test.event".to_string(),
905                payload: serde_json::json!({"ok": true}),
906            },
907        ))
908        .unwrap();
909        let text = std::fs::read_to_string(&path).unwrap();
910        assert!(text.contains("\"type\":\"custom\""));
911        assert!(text.contains("\"session_id\":\"root\""));
912    }
913
914    #[test]
915    fn tool_start_and_frame_switch_records_are_jsonl_shaped() {
916        let started = TraceRecord::new(
917            TraceContext::default().for_session("root"),
918            TraceEvent::ToolCallStarted {
919                call_id: Some("call-1".to_string()),
920                name: "read_file".to_string(),
921                args: serde_json::json!({"path": "README.md"}),
922            },
923        );
924        let completed = TraceRecord::new(
925            TraceContext::default().for_session("root"),
926            TraceEvent::TurnCompleted {
927                status: "completed".to_string(),
928                done_reason: "modelstop".to_string(),
929                agent_frame_switch: Some(TraceAgentFrameSwitch {
930                    frame_id: "frame-1".to_string(),
931                }),
932            },
933        );
934
935        let started_json = serde_json::to_value(started).unwrap();
936        assert_eq!(started_json["type"], "tool_call_started");
937        assert_eq!(started_json["call_id"], "call-1");
938
939        let completed_json = serde_json::to_value(completed).unwrap();
940        assert_eq!(completed_json["type"], "turn_completed");
941        assert_eq!(completed_json["agent_frame_switch"]["frame_id"], "frame-1");
942    }
943
944    #[test]
945    fn lashlang_execution_records_are_jsonl_shaped() {
946        let identity = TraceLashlangExecutionIdentity {
947            scope: TraceRuntimeScope::new("s1"),
948            subject: TraceRuntimeSubject::Process {
949                process_id: "p1".to_string(),
950            },
951            module_ref: "module".to_string(),
952            entry_kind: "process".to_string(),
953            entry_ref: Some("component:0".to_string()),
954            entry_name: "main".to_string(),
955        };
956        let event = TraceLashlangExecutionEvent::NodeStarted {
957            event_key: "process:p1:node:n1:1:started".to_string(),
958            identity,
959            node_id: "n1".to_string(),
960            node_kind: "resource_operation".to_string(),
961            label: "read_file".to_string(),
962            occurrence: 1,
963        };
964        let record = TraceRecord::new(
965            TraceContext::default().for_session("s1"),
966            TraceEvent::LashlangExecution { event },
967        );
968
969        let json = serde_json::to_value(&record).expect("serialize lashlang execution");
970        assert_eq!(json["type"], "lashlang_execution");
971        assert_eq!(json["event"]["kind"], "node_started");
972        assert_eq!(json["event"]["event_key"], "process:p1:node:n1:1:started");
973
974        let round_trip =
975            serde_json::from_value::<TraceRecord>(json).expect("deserialize lashlang execution");
976        assert!(matches!(
977            round_trip.event,
978            TraceEvent::LashlangExecution {
979                event: TraceLashlangExecutionEvent::NodeStarted { .. }
980            }
981        ));
982    }
983
984    #[test]
985    fn tool_completion_serializes_typed_failure_output() {
986        let record = TraceRecord::new(
987            TraceContext::default().for_session("root"),
988            TraceEvent::ToolCallCompleted {
989                call_id: Some("call-1".to_string()),
990                name: "read_file".to_string(),
991                args: serde_json::json!({"path": "missing"}),
992                output: TraceToolCallOutput {
993                    outcome: TraceToolCallOutcome::Failure(serde_json::json!({
994                        "class": "invalid_request",
995                        "code": "invalid_tool_args",
996                        "message": "bad args",
997                        "source": "runtime",
998                        "retry": { "type": "never" },
999                        "raw": { "path": "missing" }
1000                    })),
1001                    control: None,
1002                },
1003                duration_ms: 3,
1004            },
1005        );
1006
1007        let json = serde_json::to_value(record).unwrap();
1008        assert_eq!(json["type"], "tool_call_completed");
1009        assert_eq!(json["output"]["outcome"]["status"], "failure");
1010        assert_eq!(
1011            json["output"]["outcome"]["payload"]["code"],
1012            "invalid_tool_args"
1013        );
1014        assert_eq!(
1015            json["output"]["outcome"]["payload"]["raw"]["path"],
1016            "missing"
1017        );
1018    }
1019
1020    #[test]
1021    fn event_kind_matches_serialized_type_tag() {
1022        let events = [
1023            TraceEvent::SessionStarted {
1024                metadata: Default::default(),
1025            },
1026            TraceEvent::TurnStarted {
1027                metadata: Default::default(),
1028            },
1029            TraceEvent::ToolCallStarted {
1030                call_id: None,
1031                name: "read_file".to_string(),
1032                args: Value::Null,
1033            },
1034            TraceEvent::Custom {
1035                name: "x".to_string(),
1036                payload: Value::Null,
1037            },
1038        ];
1039        for event in events {
1040            let kind = event.kind();
1041            let json = serde_json::to_value(&event).expect("serialize event");
1042            assert_eq!(json["type"], kind, "kind() disagrees with serde tag");
1043        }
1044    }
1045
1046    #[test]
1047    fn jsonl_sink_creates_parent_directories() {
1048        let dir = std::env::temp_dir().join(format!("lash-trace-{}", uuid::Uuid::new_v4()));
1049        let path = dir.join("nested").join("trace.jsonl");
1050        let sink = JsonlTraceSink::new(&path);
1051        sink.append(&TraceRecord::new(
1052            TraceContext::default().for_session("root"),
1053            TraceEvent::RuntimeStreamEvent {
1054                event: TraceRuntimeStreamEvent {
1055                    sequence: 1,
1056                    elapsed_ms: 0,
1057                    event_name: "delta".to_string(),
1058                    raw_text: Some("hello".to_string()),
1059                    visible_text: Some("hello".to_string()),
1060                    item_id: None,
1061                    output_index: None,
1062                    call_id: None,
1063                    tool_name: None,
1064                    input_json: None,
1065                    usage: None,
1066                },
1067            },
1068        ))
1069        .unwrap();
1070        assert!(path.exists());
1071        let _ = std::fs::remove_dir_all(dir);
1072    }
1073}
lash_trace/lib.rs

lash_trace/
lib.rs
