tokenfold_core/
report.rs

1use serde::{Deserialize, Serialize};
2
3use crate::status::Status;
4
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
6pub struct CompressionReport {
7    pub schema_version: String,
8    pub original_tokens: usize,
9    pub compressed_tokens: usize,
10    pub saved_tokens: usize,
11    pub savings_ratio: f64, // fraction: 0.353
12    pub savings_pct: f64,   // positive percent: 35.3
13    pub estimator: EstimatorInfo,
14    pub status: Status,
15    pub mode: String,
16    pub format: String,
17    pub task_scope: String,
18    pub request_id: Option<String>,
19    /// F-055: staged `raw -> RTK -> tokenfold` accounting. `None` for the common
20    /// single-stage path; populated only by RTK-composed `wrap --rtk` runs.
21    #[serde(default)]
22    pub pipeline: Option<PipelineReport>,
23    pub quality: Option<QualityReport>,
24    pub budget: Option<BudgetReport>,
25    pub cache: Option<CacheReport>,
26    pub retrieval: Option<RetrievalReport>,
27    pub output_savings: Option<OutputSavingsReport>,
28    pub bypass: Option<BypassReport>,
29    pub command: Option<CommandReport>,
30    pub ledger: Option<LedgerReport>,
31    pub transforms: Vec<TransformReport>,
32    pub warnings: Vec<Warning>,
33}
34
35impl CompressionReport {
36    #[allow(clippy::too_many_arguments)]
37    pub fn new(
38        original_tokens: usize,
39        compressed_tokens: usize,
40        estimator: EstimatorInfo,
41        status: Status,
42        mode: String,
43        format: String,
44        task_scope: String,
45        transforms: Vec<TransformReport>,
46        warnings: Vec<Warning>,
47    ) -> Self {
48        let saved_tokens = original_tokens.saturating_sub(compressed_tokens);
49        let savings_ratio = if original_tokens == 0 {
50            0.0
51        } else {
52            saved_tokens as f64 / original_tokens as f64
53        };
54        let savings_pct = savings_ratio * 100.0;
55        Self {
56            schema_version: "1.0".to_string(),
57            original_tokens,
58            compressed_tokens,
59            saved_tokens,
60            savings_ratio,
61            savings_pct,
62            estimator,
63            status,
64            mode,
65            format,
66            task_scope,
67            request_id: None,
68            pipeline: None,
69            quality: None,
70            budget: None,
71            cache: None,
72            retrieval: None,
73            output_savings: None,
74            bypass: None,
75            command: None,
76            ledger: None,
77            transforms,
78            warnings,
79        }
80    }
81}
82
83#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
84pub struct EstimatorInfo {
85    pub backend: String,
86    pub model: Option<String>,
87    pub is_exact: bool,
88}
89
90/// F-055: separates savings and recoverability across composed stages (RTK then
91/// tokenfold) so RTK's savings are never credited to tokenfold. The top-level
92/// `original_tokens` keeps its v1 meaning — tokens *entering* `tokenfold_core`,
93/// which is the post-RTK count when composed. See INTERFACES.md §1.9 / §2.
94#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
95pub struct PipelineReport {
96    /// Pre-RTK byte count. `Some` only when a complete raw capture was observed.
97    pub raw_input_bytes: Option<usize>,
98    /// Pre-RTK token count. `Some` only when raw capture is complete.
99    pub raw_input_tokens: Option<usize>,
100    pub final_output_bytes: usize,
101    /// Equals top-level `compressed_tokens`.
102    pub final_output_tokens: usize,
103    /// Populated only when raw and final counts use the same estimator.
104    pub total_saved_tokens: Option<usize>,
105    /// `"complete"`, `"partial"`, `"unavailable"`, or `"not_applicable"`.
106    pub raw_capture: String,
107    /// `"full"`, `"tokenfold_only"`, `"none"`, or `"not_applicable"`.
108    pub upstream_recoverability: String,
109    pub stages: Vec<PipelineStageReport>,
110}
111
112/// F-055: one composed stage. Count fields are nullable because an unavailable
113/// stage or missing pre-stage capture cannot be measured honestly.
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
115pub struct PipelineStageReport {
116    /// `"rtk"` or `"tokenfold"`.
117    pub id: String,
118    pub version: Option<String>,
119    pub input_bytes: Option<usize>,
120    pub output_bytes: Option<usize>,
121    pub saved_bytes: Option<usize>,
122    pub input_tokens: Option<usize>,
123    pub output_tokens: Option<usize>,
124    pub saved_tokens: Option<usize>,
125    pub estimator: Option<EstimatorInfo>,
126    /// `"applied"`, `"passthrough"`, `"unavailable"`, `"incompatible"`, or `"failed"`.
127    pub status: String,
128    pub duration_ms: Option<f64>,
129    pub bypass_reason: Option<String>,
130    /// e.g. `"external:rtk@0.4.1"` or `"tokenfold_core"`.
131    pub provenance: String,
132    /// `"full"`, `"partial"`, `"none"`, or `"not_applicable"`.
133    pub recoverability: String,
134    pub evidence_ref: Option<String>,
135}
136
137#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
138pub struct BudgetReport {
139    pub target_tokens: Option<usize>,
140    pub protected_floor: usize,
141    pub achieved_tokens: usize,
142}
143
144#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
145pub struct QualityReport {
146    pub eval_profile_id: String,
147    pub task_scope: String,
148    pub validated_ratio_band: Option<String>,
149    /// `None` when a lossy transform ran but no fidelity-gate data was baked in at build time —
150    /// `interfaces.md`'s explicit "early dev builds before Phase 2" state. These were plain
151    /// `f64` before, which forced that state to be reported as a fabricated `0.0` ("nothing was
152    /// retained") — indistinguishable from a real, measured total-loss result. Absent data must
153    /// read as absent, not as a measurement.
154    #[serde(default)]
155    pub quality_retention: Option<f64>,
156    #[serde(default)]
157    pub contrastive_failure_rate: Option<f64>,
158    pub gate_passed: bool,
159}
160
161#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
162pub struct TransformReport {
163    pub id: String,
164    pub version: String,
165    pub tokens_before: usize,
166    pub tokens_after: usize,
167    pub saved_tokens: usize,
168    pub savings_ratio: f64,
169    pub elapsed_micros: Option<u64>,
170    pub status: TransformStatus,
171    pub skipped_reason: Option<SkippedReason>,
172    pub warnings: Vec<Warning>,
173}
174
175#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
176#[serde(rename_all = "snake_case")]
177pub enum TransformStatus {
178    Applied,
179    NoOp,
180    Skipped,
181    RolledBack,
182}
183
184#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
185#[serde(rename_all = "snake_case")]
186pub enum SkippedReason {
187    TargetAlreadyMet,
188    NotApplicableToFormat,
189    NotEnabledInMode,
190    /// The transform is enabled for this mode/format, but a lossy run (`policy.lossy`) actually
191    /// pruned the payload, and this transform restructures arrays in a way that would move a
192    /// `lossy_preserve` path off the array it names. See `pipeline::apply_transforms`, which
193    /// defers these until after the lossy stage and only skips them when pruning really applied.
194    IncompatibleWithLossy,
195    ExperimentalFlagRequired,
196    DisabledByUser,
197    WouldIncreaseTokens,
198    FilterUntrusted,
199    FilterFailedVerify,
200    BypassEnvSet,
201    UnsupportedCommandShape,
202    PipeOrHeredocNotRewritten,
203    BinaryOutputDetected,
204    UnsafeCommandPassthrough,
205}
206
207#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
208pub struct Warning {
209    pub code: WarningCode,
210    pub severity: Severity,
211    pub transform: Option<String>,
212    pub message: String,
213}
214
215#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
216#[serde(rename_all = "snake_case")]
217pub enum Severity {
218    Info,
219    Warn,
220    Critical,
221}
222
223#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
224#[serde(rename_all = "snake_case")]
225pub enum WarningCode {
226    UnreachableTarget,
227    UnredactedContentPossible,
228    SafetyDowngrade,
229    SecurityFieldAltered,
230    HeuristicBudgetUsed,
231    PrefixModified,
232}
233
234#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
235pub struct CacheReport {
236    pub boundary_kind: Option<String>,
237    pub protected_bytes: usize,
238    pub prefix_byte_identical: bool,
239    pub warnings: Vec<Warning>,
240}
241
242#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
243pub struct RetrievalReport {
244    pub store_namespace: String,
245    pub hash_algorithm: String,
246    pub marker_count: usize,
247    pub ttl_seconds: Option<u64>,
248    pub persisted_original_bytes: usize,
249    pub skipped_original_bytes: usize,
250}
251
252#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
253pub struct OutputSavingsReport {
254    pub profile: String,
255    pub estimated_output_tokens_saved: Option<usize>,
256    pub measured_output_tokens_saved: Option<usize>,
257    pub provenance: String,
258}
259
260#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
261pub struct BypassReport {
262    pub reason: String,
263    pub source: String,
264}
265
266#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
267pub struct CommandReport {
268    pub command_family: Option<String>,
269    pub child_exit_code: Option<i32>,
270    pub duration_ms: u64,
271    pub raw_output_bytes: usize,
272    pub stdout_bytes: usize,
273    pub stderr_bytes: usize,
274    pub stderr_mode: String,
275    pub stderr_truncated: bool,
276    pub compressed_output_bytes: usize,
277    pub filter_pack_id: Option<String>,
278    pub filter_version: Option<String>,
279    pub never_worse_applied: bool,
280    pub bypass_reason: Option<String>,
281}
282
283#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
284pub struct LedgerReport {
285    pub recorded: bool,
286    pub scope: Option<String>,
287    pub project_hash: Option<String>,
288    pub record_id: Option<String>,
289}
290
291#[cfg(test)]
292mod tests {
293    use super::*;
294
295    fn heuristic_estimator() -> EstimatorInfo {
296        EstimatorInfo {
297            backend: "heuristic".to_string(),
298            model: None,
299            is_exact: false,
300        }
301    }
302
303    #[test]
304    fn saved_tokens_and_ratio_are_derived_correctly() {
305        let report = CompressionReport::new(
306            18_400,
307            11_900,
308            heuristic_estimator(),
309            Status::Compressed,
310            "balanced".to_string(),
311            "plain_text".to_string(),
312            "general".to_string(),
313            vec![],
314            vec![],
315        );
316        assert_eq!(report.saved_tokens, 6_500);
317        assert!((report.savings_ratio - 0.353_260_869_565_217_4).abs() < f64::EPSILON * 10.0);
318        assert!((report.savings_pct - 35.326_086_956_521_74).abs() < 1e-9);
319        assert_eq!(report.schema_version, "1.0");
320    }
321
322    #[test]
323    fn zero_original_tokens_never_divides_by_zero() {
324        let report = CompressionReport::new(
325            0,
326            0,
327            heuristic_estimator(),
328            Status::Passthrough,
329            "balanced".to_string(),
330            "plain_text".to_string(),
331            "general".to_string(),
332            vec![],
333            vec![],
334        );
335        assert_eq!(report.saved_tokens, 0);
336        assert_eq!(report.savings_ratio, 0.0);
337        assert_eq!(report.savings_pct, 0.0);
338    }
339
340    #[test]
341    fn compressed_never_exceeding_original_keeps_saved_tokens_nonnegative() {
342        // saturating_sub guards against compressed_tokens > original_tokens (should never
343        // happen, but the report must never panic or underflow if it does).
344        let report = CompressionReport::new(
345            10,
346            15,
347            heuristic_estimator(),
348            Status::BestEffort,
349            "balanced".to_string(),
350            "plain_text".to_string(),
351            "general".to_string(),
352            vec![],
353            vec![],
354        );
355        assert_eq!(report.saved_tokens, 0);
356    }
357
358    #[test]
359    fn status_serializes_inside_report_as_snake_case() {
360        let report = CompressionReport::new(
361            100,
362            80,
363            heuristic_estimator(),
364            Status::UnreachableTarget,
365            "balanced".to_string(),
366            "plain_text".to_string(),
367            "general".to_string(),
368            vec![],
369            vec![],
370        );
371        let json = serde_json::to_value(&report).unwrap();
372        assert_eq!(json["status"], "unreachable_target");
373        assert_eq!(json["estimator"]["backend"], "heuristic");
374        assert_eq!(json["estimator"]["is_exact"], false);
375    }
376
377    #[test]
378    fn quality_report_round_trips() {
379        let quality = QualityReport {
380            eval_profile_id: "smoke-first-consumer".to_string(),
381            task_scope: "code_review".to_string(),
382            validated_ratio_band: Some("0.6-0.8".to_string()),
383            quality_retention: Some(0.975),
384            contrastive_failure_rate: Some(0.0),
385            gate_passed: true,
386        };
387        let json = serde_json::to_string(&quality).unwrap();
388        let back: QualityReport = serde_json::from_str(&json).unwrap();
389        assert_eq!(quality, back);
390    }
391
392    #[test]
393    fn quality_report_without_baked_in_gate_data_round_trips_as_absent_not_zero() {
394        let quality = QualityReport {
395            eval_profile_id: "unvalidated".to_string(),
396            task_scope: "all".to_string(),
397            validated_ratio_band: None,
398            quality_retention: None,
399            contrastive_failure_rate: None,
400            gate_passed: false,
401        };
402        let json = serde_json::to_value(&quality).unwrap();
403        assert!(
404            json["quality_retention"].is_null(),
405            "absent must not serialize as 0.0"
406        );
407        let back: QualityReport = serde_json::from_value(json).unwrap();
408        assert_eq!(quality, back);
409    }
410}
tokenfold_core/report.rs

tokenfold_core/
report.rs
