tokenfold_core/
modes.rs

1//! Canonical mode matrix: the single source of truth for which transforms run in which
2//! mode, at what ratio cap, for which task scopes and input formats. `secret_redaction` is
3//! deliberately absent from this table — it runs unconditionally before the pipeline and
4//! cannot be disabled (see `budget::CompressionPolicyBuilder::build`).
5//!
6//! `tests/fixtures/mode_matrix.toml` mirrors this table for cross-surface testing
7//! (this table is authoritative; the fixture must be kept in sync with it).
8
9use crate::budget::{CompressionMode, TaskScope};
10use crate::input::InputFormat;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum TransformId {
14    JsonMinify,
15    JsonFieldFold,
16    JsonValueDict,
17    SchemaCompaction,
18    LogFieldFold,
19    LogCompaction,
20    DiffCompaction,
21}
22
23impl TransformId {
24    pub fn as_str(&self) -> &'static str {
25        match self {
26            TransformId::JsonMinify => "json_minify",
27            TransformId::JsonFieldFold => "json_field_fold",
28            TransformId::JsonValueDict => "json_value_dict",
29            TransformId::SchemaCompaction => "schema_compaction",
30            TransformId::LogFieldFold => "log_field_fold",
31            TransformId::LogCompaction => "log_compaction",
32            TransformId::DiffCompaction => "diff_compaction",
33        }
34    }
35}
36
37#[derive(Debug, Clone, Copy)]
38pub struct ModeEntry {
39    pub transform_id: TransformId,
40    pub version: &'static str,
41    pub conservative_enabled: bool,
42    pub balanced_enabled: bool,
43    pub aggressive_enabled: bool,
44    pub experimental: bool,
45    pub max_ratio_conservative: f64,
46    pub max_ratio_balanced: f64,
47    pub max_ratio_aggressive: f64,
48    pub task_scopes: &'static [TaskScope],
49    pub applicable_formats: &'static [InputFormat],
50}
51
52impl ModeEntry {
53    pub fn enabled_for(&self, mode: CompressionMode) -> bool {
54        match mode {
55            CompressionMode::Conservative => self.conservative_enabled,
56            CompressionMode::Balanced => self.balanced_enabled,
57            CompressionMode::Aggressive => self.aggressive_enabled,
58        }
59    }
60
61    pub fn max_ratio_for(&self, mode: CompressionMode) -> f64 {
62        match mode {
63            CompressionMode::Conservative => self.max_ratio_conservative,
64            CompressionMode::Balanced => self.max_ratio_balanced,
65            CompressionMode::Aggressive => self.max_ratio_aggressive,
66        }
67    }
68
69    fn applies_to_format(&self, format: InputFormat) -> bool {
70        self.applicable_formats.contains(&format)
71    }
72}
73
74// Canonical ordered table — order here IS the pipeline execution order (lossless before lossy,
75// higher-savings before lower-savings, within each mode).
76//
77// ponytail: `table_compaction` is intentionally omitted. Tabular payloads aren't among this
78// project's dominant input types, so a table transform stays out of scope until a real consumer
79// asks for it.
80pub static ALL_ENTRIES: &[ModeEntry] = &[
81    ModeEntry {
82        transform_id: TransformId::JsonMinify,
83        version: "1.0.0",
84        conservative_enabled: true,
85        balanced_enabled: true,
86        aggressive_enabled: true,
87        experimental: false,
88        max_ratio_conservative: 1.0,
89        max_ratio_balanced: 1.0,
90        max_ratio_aggressive: 1.0,
91        task_scopes: &[TaskScope::All],
92        applicable_formats: &[
93            InputFormat::OpenAiJson,
94            InputFormat::AnthropicJson,
95            InputFormat::Json,
96        ],
97    },
98    // json_field_fold (v0.2): reversible columnar fold of arrays of homogeneous objects.
99    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but
100    // it restructures what the model sees, so it stays out of Conservative (same convention
101    // as log_compaction) and only runs on generic Json data, never on OpenAI/Anthropic
102    // message bodies (whose API shape must not change).
103    ModeEntry {
104        transform_id: TransformId::JsonFieldFold,
105        version: "1.0.0",
106        conservative_enabled: false,
107        balanced_enabled: true,
108        aggressive_enabled: true,
109        experimental: false,
110        max_ratio_conservative: 0.0,
111        max_ratio_balanced: 1.0,
112        max_ratio_aggressive: 1.0,
113        task_scopes: &[TaskScope::All],
114        applicable_formats: &[InputFormat::Json],
115    },
116    // json_value_dict (v0.2): reversible value deduplication. Runs AFTER json_field_fold so it
117    // also collapses the repeated nested values folding surfaces across rows. Lossless
118    // (round-trip gated), unrestricted ratio, out of Conservative, generic Json only.
119    ModeEntry {
120        transform_id: TransformId::JsonValueDict,
121        version: "1.0.0",
122        conservative_enabled: false,
123        balanced_enabled: true,
124        aggressive_enabled: true,
125        experimental: false,
126        max_ratio_conservative: 0.0,
127        max_ratio_balanced: 1.0,
128        max_ratio_aggressive: 1.0,
129        task_scopes: &[TaskScope::All],
130        applicable_formats: &[InputFormat::Json],
131    },
132    ModeEntry {
133        transform_id: TransformId::SchemaCompaction,
134        version: "1.0.0",
135        conservative_enabled: true,
136        balanced_enabled: true,
137        aggressive_enabled: true,
138        experimental: false,
139        max_ratio_conservative: 0.15,
140        max_ratio_balanced: 0.30,
141        max_ratio_aggressive: 0.50,
142        task_scopes: &[TaskScope::All],
143        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
144    },
145    // log_field_fold (v0.4): reversible columnar fold of TEMPLATED log lines — the log-line
146    // analogue of json_field_fold (emit each shared line skeleton once + per-line captured fields).
147    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but like
148    // log_compaction it restructures what the model sees, so it stays out of Conservative and ships
149    // behind --experimental until its fidelity gate is green (the same path json_field_fold and
150    // log_compaction took). Runs before the lossy log_compaction (lossless-before-lossy ordering).
151    ModeEntry {
152        transform_id: TransformId::LogFieldFold,
153        version: "1.0.0",
154        conservative_enabled: false,
155        balanced_enabled: false,
156        aggressive_enabled: false,
157        experimental: true,
158        max_ratio_conservative: 0.0,
159        max_ratio_balanced: 1.0,
160        max_ratio_aggressive: 1.0,
161        task_scopes: &[TaskScope::All],
162        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
163    },
164    ModeEntry {
165        transform_id: TransformId::LogCompaction,
166        version: "1.0.0",
167        // Promoted out of --experimental (Phase 5 fidelity gate, 2026-07-12): the
168        // full-lossy-promotion gate profile clears every draft fidelity threshold cleanly
169        // (quality_retention=1.0, contrastive_failure_rate=0.0, critical_token_survival_rate=1.0).
170        // conservative_enabled stays false — Conservative never runs lossy-with-evidence
171        // transforms at all, same convention table_compaction documents.
172        conservative_enabled: false,
173        balanced_enabled: true,
174        aggressive_enabled: true,
175        experimental: false,
176        max_ratio_conservative: 0.0,
177        max_ratio_balanced: 0.65, // draft; updated after Phase 2 accuracy@ratio data
178        max_ratio_aggressive: 0.75,
179        task_scopes: &[TaskScope::General, TaskScope::ChangeSummary],
180        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
181    },
182    ModeEntry {
183        transform_id: TransformId::DiffCompaction,
184        version: "1.0.0",
185        // Stays --experimental (Phase 5 fidelity gate, 2026-07-12 re-investigation): the
186        // full-lossy-promotion gate's per_variant breakdown checked the default (body-preserving,
187        // task_scope != ChangeSummary) and header-only (TaskScope::ChangeSummary) forms
188        // separately, as diff_compaction's two documented forms require, and BOTH miss the draft
189        // fidelity thresholds on their own — the bar is quality_retention >= 0.95,
190        // contrastive_failure_rate <= 0.005, critical_token_survival >= 0.99, and the default form
191        // measured quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
192        // 0.5. Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
193        // everything, critical tokens included, when no line matches a unified-diff prefix. See
194        // eval/tasks/FIXTURES.md's "Scorer status" section for the full measured breakdown.
195        conservative_enabled: false,
196        balanced_enabled: false,
197        aggressive_enabled: false,
198        experimental: true,
199        max_ratio_conservative: 0.0,
200        max_ratio_balanced: 0.60,
201        max_ratio_aggressive: 0.70,
202        task_scopes: &[TaskScope::CodeReview, TaskScope::ChangeSummary],
203        applicable_formats: &[
204            InputFormat::PlainText,
205            InputFormat::CommandOutput,
206            InputFormat::GitDiff,
207        ],
208    },
209    // v0.2+ entries (table_compaction, prose_extraction, code_digest, conversation) added
210    // here after their fidelity approval / scope decisions land.
211];
212
213/// Returns the ordered, applicable transform list for a given (mode, task_scope, format).
214/// `secret_redaction` is not part of this table: the pipeline always runs it first,
215/// unconditionally, before consulting this function.
216pub fn pipeline_for(
217    mode: CompressionMode,
218    task_scope: TaskScope,
219    format: InputFormat,
220    experimental: bool,
221    enabled_ids: &[String],
222    disabled_ids: &[String],
223) -> Vec<&'static ModeEntry> {
224    ALL_ENTRIES
225        .iter()
226        .filter(|e| {
227            let mode_enabled = e.enabled_for(mode);
228            let experimentally_enabled = e.experimental && experimental;
229            let explicitly_enabled = (!e.experimental || experimental)
230                && enabled_ids.iter().any(|id| id == e.transform_id.as_str());
231            mode_enabled || experimentally_enabled || explicitly_enabled
232        })
233        .filter(|e| !disabled_ids.iter().any(|id| id == e.transform_id.as_str()))
234        .filter(|e| e.task_scopes.contains(&TaskScope::All) || e.task_scopes.contains(&task_scope))
235        .filter(|e| e.applies_to_format(format))
236        .collect()
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    #[test]
244    fn transform_id_as_str_matches_canonical_ids() {
245        assert_eq!(TransformId::JsonMinify.as_str(), "json_minify");
246        assert_eq!(TransformId::SchemaCompaction.as_str(), "schema_compaction");
247        assert_eq!(TransformId::LogCompaction.as_str(), "log_compaction");
248        assert_eq!(TransformId::DiffCompaction.as_str(), "diff_compaction");
249    }
250
251    #[test]
252    fn conservative_mode_never_includes_experimental_lossy_transforms() {
253        let entries = pipeline_for(
254            CompressionMode::Conservative,
255            TaskScope::All,
256            InputFormat::PlainText,
257            /* experimental */ true,
258            &[],
259            &[],
260        );
261        assert!(
262            entries
263                .iter()
264                .all(|e| e.transform_id != TransformId::LogCompaction
265                    && e.transform_id != TransformId::DiffCompaction)
266        );
267    }
268
269    #[test]
270    fn balanced_mode_lossless_transforms_apply_to_openai_json() {
271        let entries = pipeline_for(
272            CompressionMode::Balanced,
273            TaskScope::All,
274            InputFormat::OpenAiJson,
275            false,
276            &[],
277            &[],
278        );
279        let ids: Vec<_> = entries.iter().map(|e| e.transform_id).collect();
280        assert!(ids.contains(&TransformId::JsonMinify));
281        assert!(ids.contains(&TransformId::SchemaCompaction));
282    }
283
284    #[test]
285    fn experimental_flag_enables_log_compaction_for_matching_task_scope() {
286        let entries = pipeline_for(
287            CompressionMode::Balanced,
288            TaskScope::General,
289            InputFormat::CommandOutput,
290            true,
291            &[],
292            &[],
293        );
294        assert!(
295            entries
296                .iter()
297                .any(|e| e.transform_id == TransformId::LogCompaction)
298        );
299    }
300
301    #[test]
302    fn log_compaction_skipped_for_non_applicable_format_even_when_experimental() {
303        let entries = pipeline_for(
304            CompressionMode::Balanced,
305            TaskScope::General,
306            InputFormat::OpenAiJson,
307            true,
308            &[],
309            &[],
310        );
311        assert!(
312            !entries
313                .iter()
314                .any(|e| e.transform_id == TransformId::LogCompaction)
315        );
316    }
317
318    #[test]
319    fn disabled_ids_remove_a_transform_even_when_otherwise_enabled() {
320        let entries = pipeline_for(
321            CompressionMode::Balanced,
322            TaskScope::All,
323            InputFormat::OpenAiJson,
324            false,
325            &[],
326            &["json_minify".to_string()],
327        );
328        assert!(
329            !entries
330                .iter()
331                .any(|e| e.transform_id == TransformId::JsonMinify)
332        );
333    }
334
335    #[test]
336    fn diff_compaction_requires_matching_task_scope_even_with_enable_flag() {
337        // enable + experimental together still respect task_scope filtering.
338        let entries = pipeline_for(
339            CompressionMode::Balanced,
340            TaskScope::Debugging,
341            InputFormat::GitDiff,
342            true,
343            &["diff_compaction".to_string()],
344            &[],
345        );
346        assert!(
347            !entries
348                .iter()
349                .any(|e| e.transform_id == TransformId::DiffCompaction)
350        );
351    }
352}
tokenfold_core/modes.rs

tokenfold_core/
modes.rs
