cu_profiler_core/
profiler.rs

1//! The profiler engine: runs scenarios through a backend, parses the logs, and
2//! assembles a [`Report`] — applying budget policies, baseline comparison,
3//! confidence scoring and diagnostics along the way.
4
5use crate::backend::ExecutionBackend;
6use crate::baseline::{BaselineComparison, BaselineStore, Fingerprint};
7use crate::budget::{self, PolicyResult};
8use crate::confidence::{self, Confidence, ConfidenceFactors};
9use crate::diagnostics::{self, Context};
10use crate::metadata::RunMetadata;
11use crate::model::{InstructionMeasurement, Measurement, Report, ScenarioReport, Status};
12use crate::parser::{self, ParseAnalysis};
13use crate::program_registry::ProgramRegistry;
14use crate::scenario::{ExpectedResult, Scenario};
15
16/// Configurable profiler engine.
17#[derive(Debug, Clone)]
18pub struct Profiler {
19    registry: ProgramRegistry,
20    config_repr: String,
21    include_raw_logs: bool,
22}
23
24impl Default for Profiler {
25    fn default() -> Self {
26        Self {
27            registry: ProgramRegistry::with_builtins(),
28            config_repr: String::new(),
29            include_raw_logs: false,
30        }
31    }
32}
33
34impl Profiler {
35    /// A profiler with the built-in program registry.
36    #[must_use]
37    pub fn new() -> Self {
38        Self::default()
39    }
40
41    /// Replace the program registry (e.g. extended from config labels).
42    #[must_use]
43    pub fn with_registry(mut self, registry: ProgramRegistry) -> Self {
44        self.registry = registry;
45        self
46    }
47
48    /// Set a string representation of the effective config, hashed into
49    /// fingerprints so baselines go stale when configuration changes.
50    #[must_use]
51    pub fn with_config_repr(mut self, repr: impl Into<String>) -> Self {
52        self.config_repr = repr.into();
53        self
54    }
55
56    /// Include raw logs in each scenario report (they can be large).
57    #[must_use]
58    pub fn include_raw_logs(mut self, yes: bool) -> Self {
59        self.include_raw_logs = yes;
60        self
61    }
62
63    /// The fingerprint of a scenario under the current config.
64    #[must_use]
65    pub fn fingerprint(&self, scenario: &Scenario) -> Fingerprint {
66        Fingerprint::new(
67            &format!("{scenario:?}"),
68            &scenario.name,
69            &self.config_repr,
70            None,
71        )
72    }
73
74    /// Run all `scenarios` through `backend`, comparing against `baseline` when
75    /// provided, and assemble a [`Report`].
76    #[must_use]
77    pub fn run(
78        &self,
79        backend: &dyn ExecutionBackend,
80        scenarios: &[Scenario],
81        baseline: Option<&BaselineStore>,
82        metadata: RunMetadata,
83    ) -> Report {
84        let reports = scenarios
85            .iter()
86            .map(|s| self.profile_one(backend, s, baseline))
87            .collect();
88        Report::new(reports, metadata)
89    }
90
91    fn profile_one(
92        &self,
93        backend: &dyn ExecutionBackend,
94        scenario: &Scenario,
95        baseline: Option<&BaselineStore>,
96    ) -> ScenarioReport {
97        // The first run drives the structural report (call tree, per-instruction,
98        // scopes). On a non-deterministic backend, extra samples add only their
99        // `total_cu` to a distribution; a deterministic backend (recorded replay)
100        // is run once — multi-sampling it would fabricate a spread of zero.
101        let output = match backend.run(scenario) {
102            Ok(output) => output,
103            Err(e) => return self.simulation_error_report(scenario, &e.to_string()),
104        };
105        let analysis = parser::analyze(&output.logs, &self.registry);
106
107        let extra = if backend.is_deterministic() {
108            0
109        } else {
110            u64::from(scenario.samples.saturating_sub(1))
111        };
112        let mut totals = vec![analysis.total_cu];
113        for _ in 0..extra {
114            // Best-effort: a sample that fails to execute or parse is skipped, not
115            // fatal — the surviving samples still yield an honest distribution.
116            if let Ok(o) = backend.run(scenario) {
117                totals.push(parser::analyze(&o.logs, &self.registry).total_cu);
118            }
119        }
120        let sample_stats = crate::model::SampleStats::from_samples(&totals);
121
122        self.assemble(
123            scenario,
124            analysis,
125            output.success,
126            output.logs,
127            baseline,
128            sample_stats,
129        )
130    }
131
132    fn assemble(
133        &self,
134        scenario: &Scenario,
135        analysis: ParseAnalysis,
136        sim_success: bool,
137        logs: Vec<String>,
138        baseline: Option<&BaselineStore>,
139        sample_stats: Option<crate::model::SampleStats>,
140    ) -> ScenarioReport {
141        // Each top-level (depth-1) invocation is one transaction instruction.
142        let per_instruction: Vec<InstructionMeasurement> = analysis
143            .call_tree
144            .children
145            .iter()
146            .enumerate()
147            .map(|(index, node)| InstructionMeasurement {
148                index,
149                program_id: node.program_id.clone(),
150                label: node.label.clone(),
151                consumed: node.units_consumed,
152            })
153            .collect();
154
155        let measurement = Measurement {
156            total_cu: analysis.total_cu,
157            consumed: analysis.total_cu,
158            requested_limit: analysis.requested_limit,
159            over_requested: analysis.over_requested,
160            cpi_count: analysis.cpi_count,
161            cpi_depth: analysis.cpi_depth,
162            unattributed_pct: analysis.unattributed_pct,
163            instrumentation_overhead_pct: None,
164            per_instruction,
165            sample_stats,
166            simulation_success: sim_success && analysis.simulation_success,
167        };
168
169        // Baseline comparison.
170        let current_fp = self.fingerprint(scenario);
171        let comparison = baseline
172            .and_then(|store| store.get(&scenario.name))
173            .map(|record| {
174                BaselineComparison::compute(
175                    record.actual_units,
176                    &record.fingerprint,
177                    &measurement,
178                    &current_fp,
179                )
180            });
181        let baseline_units = comparison
182            .as_ref()
183            .filter(|c| c.matched)
184            .map(|c| c.baseline_units);
185
186        // Budget policy.
187        let policy_results: Vec<PolicyResult> =
188            budget::evaluate(&measurement, &scenario.budget, baseline_units);
189
190        // Confidence.
191        let confidence = self.score_confidence(
192            &analysis,
193            comparison.as_ref(),
194            measurement.sample_stats.map(|s| s.cv),
195        );
196
197        // Status.
198        let status = self.derive_status(&measurement, &policy_results, scenario.expected);
199
200        // Diagnostics.
201        let ctx = Context {
202            scenario: &scenario.name,
203            measurement: &measurement,
204            policy_results: &policy_results,
205            baseline: comparison.as_ref(),
206            confidence: &confidence,
207            expected: scenario.expected,
208            scope_count: analysis.scope_marker_count,
209            log_line_count: analysis.log_line_count,
210            late_validation: analysis.validation_after_cpi,
211        };
212        let diags = diagnostics::evaluate(&ctx);
213
214        ScenarioReport {
215            name: scenario.name.clone(),
216            status,
217            measurement,
218            call_tree: Some(analysis.call_tree),
219            scopes: analysis.scopes,
220            policy_results,
221            diagnostics: diags,
222            confidence,
223            baseline_comparison: comparison,
224            parser_warnings: analysis.warnings,
225            raw_logs: self.include_raw_logs.then_some(logs),
226        }
227    }
228
229    fn score_confidence(
230        &self,
231        analysis: &ParseAnalysis,
232        comparison: Option<&BaselineComparison>,
233        sample_cv: Option<f64>,
234    ) -> Confidence {
235        // Unattributed CU only counts against confidence when the user opted
236        // into scope attribution; otherwise it is just normal program work.
237        let unattributed_pct = if analysis.scope_marker_count > 0 {
238            analysis.unattributed_pct
239        } else {
240            0.0
241        };
242        let factors = ConfidenceFactors {
243            simulation_ok: analysis.simulation_success,
244            logs_complete: analysis.logs_complete,
245            parser_warnings: analysis.warnings.len(),
246            baseline_matched: comparison.map(|c| c.matched),
247            unattributed_pct,
248            scope_markers: analysis.scope_marker_count,
249            metadata_available: true,
250            sample_cv,
251        };
252        confidence::score(&factors)
253    }
254
255    fn derive_status(
256        &self,
257        measurement: &Measurement,
258        policy_results: &[PolicyResult],
259        expected: ExpectedResult,
260    ) -> Status {
261        // An unexpected simulation outcome is a failure regardless of budgets.
262        let outcome_ok = match expected {
263            ExpectedResult::Success => measurement.simulation_success,
264            ExpectedResult::Failure => !measurement.simulation_success,
265        };
266        if !outcome_ok {
267            return Status::Fail;
268        }
269        Status::from_policy(budget::overall_status(policy_results))
270    }
271
272    fn simulation_error_report(&self, scenario: &Scenario, error: &str) -> ScenarioReport {
273        ScenarioReport {
274            name: scenario.name.clone(),
275            status: Status::Unknown,
276            measurement: Measurement {
277                simulation_success: false,
278                ..Measurement::empty()
279            },
280            call_tree: None,
281            scopes: Vec::new(),
282            policy_results: Vec::new(),
283            diagnostics: Vec::new(),
284            confidence: Confidence::unknown(format!("simulation error: {error}")),
285            baseline_comparison: None,
286            parser_warnings: vec![format!("simulation error: {error}")],
287            raw_logs: None,
288        }
289    }
290}
291
292#[cfg(test)]
293mod tests {
294    use super::*;
295    use crate::backend::RecordedLogsBackend;
296    use crate::budget::BudgetPolicy;
297
298    fn backend() -> RecordedLogsBackend {
299        let mut b = RecordedLogsBackend::new();
300        b.insert_blob(
301            "swap",
302            "Program User111 invoke [1]\n\
303             Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]\n\
304             Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3000 of 197000 compute units\n\
305             Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success\n\
306             Program User111 consumed 96000 of 200000 compute units\n\
307             Program User111 success",
308            true,
309        );
310        b
311    }
312
313    fn swap_scenario(max: u64) -> Scenario {
314        let mut s = Scenario::new("swap");
315        s.budget = BudgetPolicy {
316            absolute_max_cu: Some(max),
317            warn_at_budget_pct: Some(90.0),
318            ..Default::default()
319        };
320        s
321    }
322
323    #[test]
324    fn end_to_end_pass() {
325        let report = Profiler::new().run(
326            &backend(),
327            &[swap_scenario(200_000)],
328            None,
329            RunMetadata::recorded("0.1.0"),
330        );
331        assert_eq!(report.scenarios[0].status, Status::Pass);
332        assert_eq!(report.scenarios[0].measurement.total_cu, 96_000);
333        assert_eq!(
334            report.scenarios[0].confidence.level,
335            confidence::ConfidenceLevel::High
336        );
337
338        // The single top-level program invocation is recorded as one instruction.
339        let per = &report.scenarios[0].measurement.per_instruction;
340        assert_eq!(per.len(), 1);
341        assert_eq!(per[0].index, 0);
342        assert_eq!(per[0].program_id, "User111");
343        assert_eq!(per[0].consumed, Some(96_000));
344    }
345
346    #[test]
347    fn end_to_end_warn_near_budget() {
348        let report = Profiler::new().run(
349            &backend(),
350            &[swap_scenario(100_000)],
351            None,
352            RunMetadata::recorded("0.1.0"),
353        );
354        assert_eq!(report.scenarios[0].status, Status::Warn);
355        assert!(
356            report.scenarios[0]
357                .diagnostics
358                .iter()
359                .any(|d| d.id == "near_budget_limit")
360        );
361    }
362
363    /// A non-deterministic backend that returns a different CU figure per call,
364    /// cycling through `cus` — used to exercise multi-sampling and variance.
365    struct VaryingBackend {
366        calls: std::cell::Cell<usize>,
367        cus: Vec<u64>,
368    }
369
370    impl crate::backend::ExecutionBackend for VaryingBackend {
371        fn kind(&self) -> crate::metadata::BackendKind {
372            crate::metadata::BackendKind::Mollusk
373        }
374        fn run(&self, _scenario: &Scenario) -> crate::Result<crate::backend::SimulationOutput> {
375            let i = self.calls.get();
376            self.calls.set(i + 1);
377            let cu = self.cus[i % self.cus.len()];
378            Ok(crate::backend::SimulationOutput::success(vec![
379                "Program P invoke [1]".to_string(),
380                format!("Program P consumed {cu} of 200000 compute units"),
381                "Program P success".to_string(),
382            ]))
383        }
384    }
385
386    #[test]
387    fn multi_sample_records_variance_and_demotes_confidence() {
388        let backend = VaryingBackend {
389            calls: std::cell::Cell::new(0),
390            cus: vec![100_000, 120_000, 110_000],
391        };
392        let mut s = swap_scenario(200_000);
393        s.samples = 3;
394        let report = Profiler::new().run(&backend, &[s], None, RunMetadata::recorded("0.1.0"));
395
396        let stats = report.scenarios[0]
397            .measurement
398            .sample_stats
399            .expect("multi-sample stats present");
400        assert_eq!(stats.count, 3);
401        assert_eq!(stats.min, 100_000);
402        assert_eq!(stats.max, 120_000);
403        assert!(stats.variance > 0.0);
404        // ~7.4% CV demotes confidence below High with a variance reason.
405        assert!(report.scenarios[0].confidence.level < confidence::ConfidenceLevel::High);
406        assert!(
407            report.scenarios[0]
408                .confidence
409                .reasons
410                .iter()
411                .any(|r| r.contains("variance"))
412        );
413    }
414
415    #[test]
416    fn deterministic_backend_ignores_samples() {
417        // The recorded backend is deterministic: samples > 1 must not fabricate a
418        // distribution — there is no real run-to-run spread to report.
419        let mut s = swap_scenario(200_000);
420        s.samples = 5;
421        let report = Profiler::new().run(&backend(), &[s], None, RunMetadata::recorded("0.1.0"));
422        assert!(report.scenarios[0].measurement.sample_stats.is_none());
423    }
424
425    #[test]
426    fn missing_scenario_yields_unknown() {
427        let report = Profiler::new().run(
428            &RecordedLogsBackend::new(),
429            &[Scenario::new("ghost")],
430            None,
431            RunMetadata::recorded("0.1.0"),
432        );
433        assert_eq!(report.scenarios[0].status, Status::Unknown);
434        assert!(report.has_failures());
435    }
436}
cu_profiler_core/profiler.rs

cu_profiler_core/
profiler.rs
