<!DOCTYPE html>
<html lang="en" style="scroll-behavior: smooth;">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Tracing - Cano Documentation</title>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism-tomorrow.min.css">
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Outfit:wght@500;700&family=Fira+Code&display=swap" rel="stylesheet">
<script src="https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.6.1/mermaid.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-rust.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-toml.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-bash.min.js"></script>
<script src="script.js" defer></script>
<style>
.page-toc {
background: var(--card-bg);
border: 1px solid var(--border-color);
border-radius: 0.75rem;
padding: 1.5rem 2rem;
margin: 2rem 0 3rem;
position: relative;
}
.page-toc::before {
content: '';
position: absolute;
left: 0; top: 0; bottom: 0;
width: 3px;
background: linear-gradient(to bottom, var(--primary-color), var(--secondary-color));
border-radius: 3px 0 0 3px;
}
.page-toc summary {
font-weight: 600;
font-size: 1.1rem;
color: #fff;
cursor: pointer;
list-style: none;
display: flex;
align-items: center;
gap: 0.5rem;
user-select: none;
}
.page-toc summary::-webkit-details-marker { display: none; }
.page-toc summary::before {
content: '';
display: inline-block;
width: 6px; height: 6px;
border-right: 2px solid var(--primary-color);
border-bottom: 2px solid var(--primary-color);
transform: rotate(-45deg);
transition: transform 0.2s ease;
flex-shrink: 0;
}
.page-toc[open] summary::before { transform: rotate(45deg); }
.page-toc ol {
list-style: none;
padding: 0;
margin: 1rem 0 0;
columns: 2;
column-gap: 2rem;
}
.page-toc li {
break-inside: avoid;
margin-bottom: 0.35rem;
}
.page-toc a {
color: var(--text-color);
text-decoration: none;
font-size: 0.95rem;
display: block;
padding: 0.25rem 0.5rem;
border-radius: 0.25rem;
transition: all 0.15s ease;
}
.page-toc a:hover {
color: var(--primary-color);
background: rgba(56, 189, 248, 0.07);
}
.page-toc .toc-sub { padding-left: 1.25rem; font-size: 0.9rem; opacity: 0.8; }
.main-content h2[id],
.main-content h3[id] { scroll-margin-top: 2rem; position: relative; }
.main-content h2[id] a.anchor-link,
.main-content h3[id] a.anchor-link {
position: absolute;
left: -1.5rem;
color: var(--border-color);
font-weight: 400;
opacity: 0;
text-decoration: none;
transition: opacity 0.15s ease;
}
.main-content h2[id]:hover a.anchor-link,
.main-content h3[id]:hover a.anchor-link {
opacity: 1;
color: var(--primary-color);
}
.feature-banner {
background: linear-gradient(135deg, rgba(129, 140, 248, 0.12), rgba(56, 189, 248, 0.12));
border: 1px solid rgba(129, 140, 248, 0.3);
border-radius: 0.75rem;
padding: 1.25rem 1.5rem;
margin: 0 0 2.5rem;
display: flex;
align-items: flex-start;
gap: 1rem;
}
.feature-banner .banner-icon {
font-size: 1.5rem;
line-height: 1;
flex-shrink: 0;
margin-top: 0.1rem;
}
.feature-banner .banner-content p { margin-bottom: 0.5rem; }
.feature-banner .banner-content p:last-child { margin-bottom: 0; }
.feature-banner code {
background: rgba(255, 255, 255, 0.08);
font-size: 0.85rem;
}
.feature-banner strong { color: #fff; }
.callout {
border-radius: 0.75rem;
padding: 1.25rem 1.5rem;
margin: 1.5rem 0;
border: 1px solid;
}
.callout-info {
background: rgba(56, 189, 248, 0.08);
border-color: rgba(56, 189, 248, 0.25);
}
.callout-info .callout-title { color: var(--primary-color); }
.callout .callout-title { font-weight: 600; margin-bottom: 0.5rem; }
.callout p:last-child { margin-bottom: 0; }
.trace-output pre {
position: relative;
border-left: 3px solid var(--secondary-color);
}
.trace-output pre::before {
content: 'output';
position: absolute;
top: 0.5rem;
right: 0.75rem;
font-size: 0.7rem;
font-weight: 600;
letter-spacing: 0.06em;
text-transform: uppercase;
color: var(--secondary-color);
opacity: 0.6;
font-family: 'Inter', sans-serif;
}
@media (max-width: 768px) {
.page-toc ol { columns: 1; }
.page-toc { padding: 1.25rem 1.25rem; }
.feature-banner { flex-direction: column; gap: 0.5rem; padding: 1rem 1.25rem; }
.callout { padding: 1rem 1.25rem; }
.main-content h2[id] a.anchor-link,
.main-content h3[id] a.anchor-link { display: none; }
}
</style>
</head>
<body>
<button id="menu-toggle" class="menu-toggle" aria-label="Toggle navigation" aria-expanded="false">☰</button>
<div class="sidebar-overlay"></div>
<nav class="sidebar" role="navigation" aria-label="Main navigation">
<a href="index.html" class="logo">
<img src="logo.png" alt="" style="height: 24px; vertical-align: middle; margin-right: 8px;">
Cano
</a>
<ul class="nav-links">
<li><a href="index.html">Home</a></li>
<li><a href="task.html">Tasks</a></li>
<li><a href="nodes.html">Nodes</a></li>
<li><a href="workflows.html">Workflows</a></li>
<li><a href="store.html">Store</a></li>
<li><a href="scheduler.html">Scheduler</a></li>
<li><a href="tracing.html" class="active">Tracing</a></li>
</ul>
<div class="sidebar-footer">
<span class="version-badge">v0.8.0</span>
<div class="sidebar-links">
<a href="https://github.com/nassor/cano" title="GitHub Repository" aria-label="GitHub">GitHub</a>
<a href="https://crates.io/crates/cano" title="Crates.io" aria-label="Crates.io">Crates.io</a>
<a href="https://docs.rs/cano" title="API Documentation" aria-label="API Docs">Docs.rs</a>
</div>
</div>
</nav>
<main class="main-content">
<h1>Tracing</h1>
<p class="subtitle">Comprehensive observability for your workflows.</p>
<div class="feature-banner">
<div class="banner-icon" aria-hidden="true">⚙️</div>
<div class="banner-content">
<p><strong>Feature flag required</strong> -- Tracing is behind the <code>tracing</code> feature gate.
Enable it with <code>features = ["tracing"]</code> or <code>features = ["all"]</code> in your
<code>Cargo.toml</code>. Zero overhead when disabled.</p>
</div>
</div>
<details class="page-toc" open>
<summary>On this page</summary>
<ol>
<li><a href="#setup">Setup</a></li>
<li><a href="#what-gets-traced">What Gets Traced</a></li>
<li><a href="#error-tracing">Error Tracing</a></li>
<li class="toc-sub"><a href="#filtering">Filtering Trace Output</a></li>
<li><a href="#scheduler-tracing">Scheduler Tracing</a></li>
<li><a href="#custom-spans">Custom Spans</a></li>
<li><a href="#custom-instrumentation">Custom Instrumentation</a></li>
<li><a href="#example-output">Example Output</a></li>
<li><a href="#full-example">Full Example</a></li>
</ol>
</details>
<p>
Cano provides comprehensive observability through the optional <code>tracing</code> feature using the
<a href="https://docs.rs/tracing/latest/tracing/" target="_blank">tracing</a> library.
All tracing instrumentation is behind conditional compilation, so it adds zero overhead when disabled.
</p>
<h2 id="setup"><a href="#setup" class="anchor-link" aria-hidden="true">#</a>Setup</h2>
<p>Enable the <code>tracing</code> feature flag in your <code>Cargo.toml</code>. You can also use
<code>features = ["all"]</code> to enable both <code>tracing</code> and <code>scheduler</code> at once.</p>
<pre><code class="language-toml">[dependencies]
cano = { version = "0.8", features = ["tracing"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# Or enable everything (tracing + scheduler):
# cano = { version = "0.8", features = ["all"] }</code></pre>
<h3 id="basic-init"><a href="#basic-init" class="anchor-link" aria-hidden="true">#</a>Basic Initialization</h3>
<p>For quick setup during development, use the default formatter.</p>
<pre><code class="language-rust">use tracing_subscriber;
// Simple setup for development
tracing_subscriber::fmt::init();</code></pre>
<h3 id="production-setup"><a href="#production-setup" class="anchor-link" aria-hidden="true">#</a>Production Setup with Environment Filter</h3>
<p>For production use, configure an environment filter so you can control log levels
at runtime via the <code>RUST_LOG</code> environment variable.</p>
<pre><code class="language-rust">use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
// Production setup with env filter
tracing_subscriber::registry()
.with(tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "info".into()))
.with(tracing_subscriber::fmt::layer())
.init();</code></pre>
<h2 id="what-gets-traced"><a href="#what-gets-traced" class="anchor-link" aria-hidden="true">#</a>What Gets Traced</h2>
<div class="card-grid">
<div class="card">
<h3>Workflow Level</h3>
<p>Orchestration start/completion, state transitions, final states.</p>
</div>
<div class="card">
<h3>Task/Node Level</h3>
<p>Execution attempts, retry logic, delays, success/failure outcomes.</p>
</div>
<div class="card">
<h3>Scheduler Level</h3>
<p>Workflow scheduling, concurrent execution, run counts, durations.</p>
</div>
</div>
<h2 id="error-tracing"><a href="#error-tracing" class="anchor-link" aria-hidden="true">#</a>Error Tracing</h2>
<p>
When the <code>tracing</code> feature is enabled, Cano automatically instruments error paths
at appropriate severity levels so you can diagnose failures without adding custom logging.
</p>
<div class="card-stack">
<div class="card">
<h3 id="failed-attempts"><a href="#failed-attempts" class="anchor-link" aria-hidden="true">#</a>Failed Task Attempts</h3>
<p>Each failed attempt is logged as a <strong>warning</strong> with the error details and current
attempt number. This lets you see transient failures that are recovered by retry logic.</p>
<pre><code class="language-bash">WARN task_attempt{attempt=2 max_attempts=4}: Task execution failed, will retry error="connection timeout"</code></pre>
</div>
<div class="card">
<h3 id="retry-exhaustion"><a href="#retry-exhaustion" class="anchor-link" aria-hidden="true">#</a>Retry Exhaustion</h3>
<p>When all retry attempts are exhausted, the final failure is logged as an <strong>error</strong>
with the total attempt count. This indicates a permanent failure that bubbles up as
<code>CanoError</code>.</p>
<pre><code class="language-bash">ERROR task_attempt{attempt=4 max_attempts=4}: Task execution failed after all retry attempts error="connection timeout"</code></pre>
</div>
<div class="card">
<h3 id="workflow-errors"><a href="#workflow-errors" class="anchor-link" aria-hidden="true">#</a>Workflow-Level Errors</h3>
<p>Workflow orchestration traces include the current state context, so errors are always
associated with the state that produced them.</p>
<pre><code class="language-bash">INFO workflow_orchestrate: Starting workflow execution initial_state=FetchData
ERROR workflow_orchestrate: Task failed in state FetchData after exhausting retries</code></pre>
</div>
</div>
<h3 id="filtering"><a href="#filtering" class="anchor-link" aria-hidden="true">#</a>Filtering Trace Output</h3>
<p>Use the <code>RUST_LOG</code> environment variable to control which modules emit trace output.
This is especially useful in production to reduce noise.</p>
<pre><code class="language-bash"># Show only Cano debug logs
RUST_LOG=cano=debug cargo run
# Show Cano info + your app's debug logs
RUST_LOG=cano=info,my_app=debug cargo run
# Show retry-related details only
RUST_LOG=cano::task=debug cargo run
# Silence everything except errors
RUST_LOG=error cargo run</code></pre>
<h2 id="scheduler-tracing"><a href="#scheduler-tracing" class="anchor-link" aria-hidden="true">#</a>Scheduler Tracing</h2>
<p>
When both the <code>scheduler</code> and <code>tracing</code> features are enabled, the scheduler
produces trace output for workflow lifecycle events. You can enable both with
<code>features = ["all"]</code>.
</p>
<div class="card-stack">
<div class="card">
<h3 id="scheduler-workflow-exec"><a href="#scheduler-workflow-exec" class="anchor-link" aria-hidden="true">#</a>Workflow Execution</h3>
<p>The scheduler traces when each managed workflow starts and completes, including the
workflow identifier and scheduling trigger type.</p>
</div>
<div class="card">
<h3 id="scheduler-retry"><a href="#scheduler-retry" class="anchor-link" aria-hidden="true">#</a>Retry Attempts</h3>
<p>Retry delay durations are logged at debug level, so you can see the backoff progression
for failing tasks within scheduled workflows.</p>
</div>
<div class="card">
<h3 id="scheduler-split"><a href="#scheduler-split" class="anchor-link" aria-hidden="true">#</a>Split Task Execution</h3>
<p>When a scheduled workflow uses split/join, each parallel task is traced with its task ID
within a <code>split_task</code> span, making it straightforward to correlate concurrent
execution in log output.</p>
</div>
</div>
<pre><code class="language-toml"># Enable both scheduler and tracing
[dependencies]
cano = { version = "0.8", features = ["all"] }</code></pre>
<h2 id="custom-spans"><a href="#custom-spans" class="anchor-link" aria-hidden="true">#</a>Custom Spans</h2>
<p>Attach custom tracing spans to your workflows to include business-specific context.
The span wraps all trace output generated during that workflow's execution.</p>
<pre><code class="language-rust">use tracing::info_span;
use cano::prelude::*;
// Create workflow with custom tracing span
let workflow_span = info_span!(
"user_data_processing",
user_id = "12345",
batch_id = "batch_001"
);
let store = MemoryStore::new();
let workflow = Workflow::new(store)
.with_tracing_span(workflow_span)
.register(MyState::Start, MyProcessorNode)
.add_exit_state(MyState::Complete);</code></pre>
<h2 id="custom-instrumentation"><a href="#custom-instrumentation" class="anchor-link" aria-hidden="true">#</a>Custom Instrumentation</h2>
<p>You can add your own tracing to Task and Node implementations using the
<code>tracing</code> crate's macros directly. This is useful for adding domain-specific
context that Cano's built-in instrumentation does not cover.</p>
<h3 id="instrument-task"><a href="#instrument-task" class="anchor-link" aria-hidden="true">#</a>Instrumenting a Task</h3>
<pre><code class="language-rust">use cano::prelude::*;
use tracing::{info, warn};
#[derive(Clone)]
struct PaymentTask;
#[async_trait]
impl Task<AppState> for PaymentTask {
async fn run(&self, store: &MemoryStore) -> Result<TaskResult<AppState>, CanoError> {
let order_id: String = store.get("order_id")?;
info!(order_id = %order_id, "Processing payment");
match process_payment(&order_id).await {
Ok(receipt) => {
info!(order_id = %order_id, receipt = %receipt, "Payment succeeded");
store.put("receipt", receipt)?;
Ok(TaskResult::Single(AppState::Confirm))
}
Err(e) => {
warn!(order_id = %order_id, error = %e, "Payment failed");
Err(CanoError::task_execution(format!("payment failed: {e}")))
}
}
}
}</code></pre>
<h3 id="instrument-node"><a href="#instrument-node" class="anchor-link" aria-hidden="true">#</a>Instrumenting a Node</h3>
<pre><code class="language-rust">use cano::prelude::*;
use tracing::{info, debug, instrument};
#[derive(Clone)]
struct DataEnrichmentNode {
source: String,
}
#[async_trait]
impl Node<PipelineState> for DataEnrichmentNode {
type PrepResult = Vec<String>;
type ExecResult = Vec<String>;
#[instrument(skip(self, store), fields(source = %self.source))]
async fn prep(&self, store: &MemoryStore) -> Result<Self::PrepResult, CanoError> {
let keys: Vec<String> = store.get("pending_keys")?;
debug!(count = keys.len(), "Loaded keys for enrichment");
Ok(keys)
}
async fn exec(&self, keys: Self::PrepResult) -> Self::ExecResult {
info!(count = keys.len(), source = %self.source, "Enriching records");
// ... enrichment logic
keys
}
async fn post(
&self,
store: &MemoryStore,
results: Self::ExecResult,
) -> Result<PipelineState, CanoError> {
info!(enriched = results.len(), "Enrichment complete");
store.put("enriched_data", results)?;
Ok(PipelineState::Validate)
}
}</code></pre>
<h2 id="example-output"><a href="#example-output" class="anchor-link" aria-hidden="true">#</a>Example Output</h2>
<p>Running with <code>RUST_LOG=info</code> produces structured logs:</p>
<div class="trace-output">
<pre><code class="language-bash">INFO user_data_processing{user_id="12345"}: Starting workflow orchestration
INFO user_data_processing{user_id="12345"}:task_execution: Starting task execution
INFO user_data_processing{user_id="12345"}:task_attempt{attempt=1}: Node execution completed success=true
INFO user_data_processing{user_id="12345"}:task_attempt{attempt=1}: processor_id=basic_processor input_records=3: Data processing completed
INFO user_data_processing{user_id="12345"}: Workflow completed successfully</code></pre>
</div>
<h2 id="full-example"><a href="#full-example" class="anchor-link" aria-hidden="true">#</a>Full Example</h2>
<pre><code class="language-rust">use cano::prelude::*;
use async_trait::async_trait;
use tracing::{info, info_span};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
enum State {
Start,
Complete,
}
#[derive(Clone)]
struct ProcessOrderNode;
#[async_trait]
impl Task<State> for ProcessOrderNode {
async fn run(&self, _store: &MemoryStore) -> Result<TaskResult<State>, CanoError> {
info!("Processing order...");
Ok(TaskResult::Single(State::Complete))
}
}
#[tokio::main]
async fn main() -> Result<(), CanoError> {
// 1. Setup Subscriber with env filter
tracing_subscriber::registry()
.with(tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "info".into()))
.with(tracing_subscriber::fmt::layer())
.init();
let store = MemoryStore::new();
// 2. Create Workflow with Custom Span
// This span will wrap all logs generated by this workflow
let workflow_span = info_span!(
"order_processing",
order_id = "ORD-2025-001",
customer = "Acme Corp"
);
let workflow = Workflow::new(store)
.register(State::Start, ProcessOrderNode)
.add_exit_state(State::Complete)
.with_tracing_span(workflow_span); // Attach span
// 3. Run
info!("Submitting order...");
workflow.orchestrate(State::Start).await?;
Ok(())
}</code></pre>
</main>
</body>
</html>
