ik_llama_cpp_2/
timing.rs

1//! Safe wrapper around `llama_timings`.
2use std::fmt::{Debug, Display, Formatter};
3
4use crate::context::LlamaContext;
5
6/// A wrapper around `llama_timings`.
7#[derive(Debug, Clone, Copy)]
8pub struct LlamaTimings {
9    pub(crate) timings: ik_llama_cpp_sys::llama_timings,
10}
11
12impl LlamaTimings {
13    /// Create a new `LlamaTimings`.
14    /// ```
15    /// # use ik_llama_cpp_2::timing::LlamaTimings;
16    /// let timings = LlamaTimings::new(1.0, 10.0, 2.0, 0.5, 3.0, 4.0, 7, 5, 6);
17    /// let timings_str = "load time = 2.00 ms
18    /// prompt eval time = 3.00 ms / 5 tokens (0.60 ms per token, 1666.67 tokens per second)
19    /// eval time = 4.00 ms / 6 runs (0.67 ms per token, 1500.00 tokens per second)\n";
20    /// assert_eq!(timings_str, format!("{}", timings));
21    /// ```
22    #[allow(clippy::too_many_arguments)]
23    #[must_use]
24    pub fn new(
25        t_start_ms: f64,
26        t_end_ms: f64,
27        t_load_ms: f64,
28        t_sample_ms: f64,
29        t_p_eval_ms: f64,
30        t_eval_ms: f64,
31        n_sample: i32,
32        n_p_eval: i32,
33        n_eval: i32,
34    ) -> Self {
35        Self {
36            timings: ik_llama_cpp_sys::llama_timings {
37                t_start_ms,
38                t_end_ms,
39                t_load_ms,
40                t_sample_ms,
41                t_p_eval_ms,
42                t_eval_ms,
43                n_sample,
44                n_p_eval,
45                n_eval,
46            },
47        }
48    }
49
50    /// Get the start time in milliseconds.
51    #[must_use]
52    pub fn t_start_ms(&self) -> f64 {
53        self.timings.t_start_ms
54    }
55
56    /// Get the end time in milliseconds.
57    #[must_use]
58    pub fn t_end_ms(&self) -> f64 {
59        self.timings.t_end_ms
60    }
61
62    /// Get the load time in milliseconds.
63    #[must_use]
64    pub fn t_load_ms(&self) -> f64 {
65        self.timings.t_load_ms
66    }
67
68    /// Get the sampling time in milliseconds.
69    #[must_use]
70    pub fn t_sample_ms(&self) -> f64 {
71        self.timings.t_sample_ms
72    }
73
74    /// Get the prompt evaluation time in milliseconds.
75    #[must_use]
76    pub fn t_p_eval_ms(&self) -> f64 {
77        self.timings.t_p_eval_ms
78    }
79
80    /// Get the evaluation time in milliseconds.
81    #[must_use]
82    pub fn t_eval_ms(&self) -> f64 {
83        self.timings.t_eval_ms
84    }
85
86    /// Get the number of sampling evaluations.
87    #[must_use]
88    pub fn n_sample(&self) -> i32 {
89        self.timings.n_sample
90    }
91
92    /// Get the number of prompt evaluations.
93    #[must_use]
94    pub fn n_p_eval(&self) -> i32 {
95        self.timings.n_p_eval
96    }
97
98    /// Get the number of evaluations.
99    #[must_use]
100    pub fn n_eval(&self) -> i32 {
101        self.timings.n_eval
102    }
103
104    /// Set the start time in milliseconds.
105    pub fn set_t_start_ms(&mut self, t_start_ms: f64) {
106        self.timings.t_start_ms = t_start_ms;
107    }
108
109    /// Set the end time in milliseconds.
110    pub fn set_t_end_ms(&mut self, t_end_ms: f64) {
111        self.timings.t_end_ms = t_end_ms;
112    }
113
114    /// Set the load time in milliseconds.
115    pub fn set_t_load_ms(&mut self, t_load_ms: f64) {
116        self.timings.t_load_ms = t_load_ms;
117    }
118
119    /// Set the sampling time in milliseconds.
120    pub fn set_t_sample_ms(&mut self, t_sample_ms: f64) {
121        self.timings.t_sample_ms = t_sample_ms;
122    }
123
124    /// Set the prompt evaluation time in milliseconds.
125    pub fn set_t_p_eval_ms(&mut self, t_p_eval_ms: f64) {
126        self.timings.t_p_eval_ms = t_p_eval_ms;
127    }
128
129    /// Set the evaluation time in milliseconds.
130    pub fn set_t_eval_ms(&mut self, t_eval_ms: f64) {
131        self.timings.t_eval_ms = t_eval_ms;
132    }
133
134    /// Set the number of sampling evaluations.
135    pub fn set_n_sample(&mut self, n_sample: i32) {
136        self.timings.n_sample = n_sample;
137    }
138
139    /// Set the number of prompt evaluations.
140    pub fn set_n_p_eval(&mut self, n_p_eval: i32) {
141        self.timings.n_p_eval = n_p_eval;
142    }
143
144    /// Set the number of evaluations.
145    pub fn set_n_eval(&mut self, n_eval: i32) {
146        self.timings.n_eval = n_eval;
147    }
148}
149
150impl Display for LlamaTimings {
151    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
152        writeln!(f, "load time = {:.2} ms", self.t_load_ms())?;
153        writeln!(
154            f,
155            "prompt eval time = {:.2} ms / {} tokens ({:.2} ms per token, {:.2} tokens per second)",
156            self.t_p_eval_ms(),
157            self.n_p_eval(),
158            self.t_p_eval_ms() / f64::from(self.n_p_eval()),
159            1e3 / self.t_p_eval_ms() * f64::from(self.n_p_eval())
160        )?;
161        writeln!(
162            f,
163            "eval time = {:.2} ms / {} runs ({:.2} ms per token, {:.2} tokens per second)",
164            self.t_eval_ms(),
165            self.n_eval(),
166            self.t_eval_ms() / f64::from(self.n_eval()),
167            1e3 / self.t_eval_ms() * f64::from(self.n_eval())
168        )?;
169        Ok(())
170    }
171}
172
173impl LlamaContext<'_> {
174    /// Returns the timings for the context.
175    pub fn timings(&self) -> LlamaTimings {
176        let timings = unsafe { ik_llama_cpp_sys::llama_get_timings(self.context.as_ptr()) };
177        LlamaTimings { timings }
178    }
179
180    /// Reset the timings for the context.
181    pub fn reset_timings(&mut self) {
182        unsafe { ik_llama_cpp_sys::llama_reset_timings(self.context.as_ptr()) }
183    }
184}
ik_llama_cpp_2/timing.rs

ik_llama_cpp_2/
timing.rs
