apr-cli 0.32.0

CLI tool for APR model inspection, debugging, and operations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Example: Using apr serve with X-Trace-Level header for debugging
//!
//! This example demonstrates how to use the apr serve endpoint with tracing
//! enabled to debug inference performance.
//!
//! # Running the server
//! ```bash
//! apr serve /path/to/model.gguf --port 8080
//! ```
//!
//! # Using tracing headers
//! ```bash
//! # Brick-level tracing (token operations)
//! curl -X POST http://localhost:8080/v1/chat/completions \
//!   -H "Content-Type: application/json" \
//!   -H "X-Trace-Level: brick" \
//!   -d '{"model":"default","messages":[{"role":"user","content":"Hi"}],"max_tokens":5}'
//!
//! # Step-level tracing (forward pass steps)
//! curl -X POST http://localhost:8080/v1/chat/completions \
//!   -H "Content-Type: application/json" \
//!   -H "X-Trace-Level: step" \
//!   -d '{"model":"default","messages":[{"role":"user","content":"Hi"}],"max_tokens":5}'
//!
//! # Layer-level tracing (per-layer timing)
//! curl -X POST http://localhost:8080/v1/chat/completions \
//!   -H "Content-Type: application/json" \
//!   -H "X-Trace-Level: layer" \
//!   -d '{"model":"default","messages":[{"role":"user","content":"Hi"}],"max_tokens":5}'
//! ```
//!
//! # Trace Response Format
//!
//! When tracing is enabled, the response includes additional trace fields:
//! ```json
//! {
//!   "id": "chatcmpl-...",
//!   "choices": [...],
//!   "usage": {...},
//!   "brick_trace": {
//!     "level": "brick",
//!     "operations": 5,
//!     "total_time_us": 12345,
//!     "breakdown": [
//!       {"name": "token_0", "time_us": 2469},
//!       {"name": "token_1", "time_us": 2469},
//!       ...
//!     ]
//!   }
//! }
//! ```
//!
//! # Without tracing
//!
//! When no X-Trace-Level header is provided, the response is a clean
//! OpenAI-compatible format with no trace fields.

use std::process::Command;

fn main() {
    println!("APR Serve Tracing Example");
    println!("=========================");
    println!();
    println!("This example demonstrates the X-Trace-Level header for debugging inference.");
    println!();
    println!("Start the server:");
    println!("  apr serve /path/to/model.gguf --port 8080");
    println!();
    println!("Then use one of these trace levels:");
    println!();
    println!("1. Brick-level (token operations):");
    println!(r#"   curl -H "X-Trace-Level: brick" http://localhost:8080/v1/chat/completions ..."#);
    println!();
    println!("2. Step-level (forward pass steps):");
    println!(r#"   curl -H "X-Trace-Level: step" http://localhost:8080/v1/chat/completions ..."#);
    println!();
    println!("3. Layer-level (per-layer timing):");
    println!(r#"   curl -H "X-Trace-Level: layer" http://localhost:8080/v1/chat/completions ..."#);
    println!();

    // Check if server is running
    let health = Command::new("curl")
        .args(["-s", "http://localhost:8080/health"])
        .output();

    match health {
        Ok(output) if output.status.success() => {
            let response = String::from_utf8_lossy(&output.stdout);
            println!("Server health: {response}");
        }
        _ => {
            println!("Server not running. Start with: apr serve <model> --port 8080");
        }
    }
}