tokenfold_core/
modes.rs

1//! Canonical mode matrix: the single source of truth for which transforms run in which
2//! mode, at what ratio cap, for which task scopes and input formats. `secret_redaction` is
3//! deliberately absent from this table — it runs unconditionally before the pipeline and
4//! cannot be disabled (see `budget::CompressionPolicyBuilder::build`).
5//!
6//! `tests/fixtures/mode_matrix.toml` mirrors this table for cross-surface testing
7//! (INTERFACES.md Part 2 is the authoritative reference for both).
8
9use crate::budget::{CompressionMode, TaskScope};
10use crate::input::InputFormat;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum TransformId {
14    JsonMinify,
15    JsonFieldFold,
16    JsonValueDict,
17    SchemaCompaction,
18    LogFieldFold,
19    LogCompaction,
20    DiffCompaction,
21}
22
23impl TransformId {
24    pub fn as_str(&self) -> &'static str {
25        match self {
26            TransformId::JsonMinify => "json_minify",
27            TransformId::JsonFieldFold => "json_field_fold",
28            TransformId::JsonValueDict => "json_value_dict",
29            TransformId::SchemaCompaction => "schema_compaction",
30            TransformId::LogFieldFold => "log_field_fold",
31            TransformId::LogCompaction => "log_compaction",
32            TransformId::DiffCompaction => "diff_compaction",
33        }
34    }
35}
36
37#[derive(Debug, Clone, Copy)]
38pub struct ModeEntry {
39    pub transform_id: TransformId,
40    pub version: &'static str,
41    pub conservative_enabled: bool,
42    pub balanced_enabled: bool,
43    pub aggressive_enabled: bool,
44    pub experimental: bool,
45    pub max_ratio_conservative: f64,
46    pub max_ratio_balanced: f64,
47    pub max_ratio_aggressive: f64,
48    pub task_scopes: &'static [TaskScope],
49    pub applicable_formats: &'static [InputFormat],
50}
51
52impl ModeEntry {
53    pub fn enabled_for(&self, mode: CompressionMode) -> bool {
54        match mode {
55            CompressionMode::Conservative => self.conservative_enabled,
56            CompressionMode::Balanced => self.balanced_enabled,
57            CompressionMode::Aggressive => self.aggressive_enabled,
58        }
59    }
60
61    pub fn max_ratio_for(&self, mode: CompressionMode) -> f64 {
62        match mode {
63            CompressionMode::Conservative => self.max_ratio_conservative,
64            CompressionMode::Balanced => self.max_ratio_balanced,
65            CompressionMode::Aggressive => self.max_ratio_aggressive,
66        }
67    }
68
69    fn applies_to_format(&self, format: InputFormat) -> bool {
70        self.applicable_formats.contains(&format)
71    }
72}
73
74// Canonical ordered table — order here IS the pipeline execution order (INTERFACES.md Part 2:
75// lossless before lossy, higher-savings before lower-savings, within each mode).
76//
77// ponytail: `table_compaction` is intentionally omitted. The First Consumer worksheet
78// (plan.md) doesn't name tables among the dominant payload types, so F-019 stays out of
79// scope until a consumer worksheet asks for it (roadmap.md F-019 dependency).
80pub static ALL_ENTRIES: &[ModeEntry] = &[
81    ModeEntry {
82        transform_id: TransformId::JsonMinify,
83        version: "1.0.0",
84        conservative_enabled: true,
85        balanced_enabled: true,
86        aggressive_enabled: true,
87        experimental: false,
88        max_ratio_conservative: 1.0,
89        max_ratio_balanced: 1.0,
90        max_ratio_aggressive: 1.0,
91        task_scopes: &[TaskScope::All],
92        applicable_formats: &[
93            InputFormat::OpenAiJson,
94            InputFormat::AnthropicJson,
95            InputFormat::Json,
96        ],
97    },
98    // json_field_fold (v0.2): reversible columnar fold of arrays of homogeneous objects.
99    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but
100    // it restructures what the model sees, so it stays out of Conservative (same convention
101    // as log_compaction) and only runs on generic Json data, never on OpenAI/Anthropic
102    // message bodies (whose API shape must not change).
103    ModeEntry {
104        transform_id: TransformId::JsonFieldFold,
105        version: "1.0.0",
106        conservative_enabled: false,
107        balanced_enabled: true,
108        aggressive_enabled: true,
109        experimental: false,
110        max_ratio_conservative: 0.0,
111        max_ratio_balanced: 1.0,
112        max_ratio_aggressive: 1.0,
113        task_scopes: &[TaskScope::All],
114        applicable_formats: &[InputFormat::Json],
115    },
116    // json_value_dict (v0.2): reversible value deduplication. Runs AFTER json_field_fold so it
117    // also collapses the repeated nested values folding surfaces across rows. Lossless
118    // (round-trip gated), unrestricted ratio, out of Conservative, generic Json only.
119    ModeEntry {
120        transform_id: TransformId::JsonValueDict,
121        version: "1.0.0",
122        conservative_enabled: false,
123        balanced_enabled: true,
124        aggressive_enabled: true,
125        experimental: false,
126        max_ratio_conservative: 0.0,
127        max_ratio_balanced: 1.0,
128        max_ratio_aggressive: 1.0,
129        task_scopes: &[TaskScope::All],
130        applicable_formats: &[InputFormat::Json],
131    },
132    ModeEntry {
133        transform_id: TransformId::SchemaCompaction,
134        version: "1.0.0",
135        conservative_enabled: true,
136        balanced_enabled: true,
137        aggressive_enabled: true,
138        experimental: false,
139        max_ratio_conservative: 0.15,
140        max_ratio_balanced: 0.30,
141        max_ratio_aggressive: 0.50,
142        task_scopes: &[TaskScope::All],
143        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
144    },
145    // log_field_fold (v0.4): reversible columnar fold of TEMPLATED log lines — the log-line
146    // analogue of json_field_fold (emit each shared line skeleton once + per-line captured fields).
147    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but like
148    // log_compaction it restructures what the model sees, so it stays out of Conservative and ships
149    // behind --experimental until its fidelity gate is green (the same path json_field_fold and
150    // log_compaction took). Runs before the lossy log_compaction (lossless-before-lossy ordering).
151    ModeEntry {
152        transform_id: TransformId::LogFieldFold,
153        version: "1.0.0",
154        conservative_enabled: false,
155        balanced_enabled: false,
156        aggressive_enabled: false,
157        experimental: true,
158        max_ratio_conservative: 0.0,
159        max_ratio_balanced: 1.0,
160        max_ratio_aggressive: 1.0,
161        task_scopes: &[TaskScope::All],
162        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
163    },
164    ModeEntry {
165        transform_id: TransformId::LogCompaction,
166        version: "1.0.0",
167        // Promoted out of --experimental (roadmap.md Phase 5 Task 9, 2026-07-12): the
168        // full-lossy-promotion fidelity gate clears every D-005 draft threshold cleanly
169        // (quality_retention=1.0, contrastive_failure_rate=0.0, critical_token_survival_rate=1.0).
170        // conservative_enabled stays false — per plan.md's mode table, Conservative never runs
171        // lossy-with-evidence transforms at all, same convention table_compaction documents.
172        conservative_enabled: false,
173        balanced_enabled: true,
174        aggressive_enabled: true,
175        experimental: false,
176        max_ratio_conservative: 0.0,
177        max_ratio_balanced: 0.65, // draft; updated after Phase 2 accuracy@ratio data
178        max_ratio_aggressive: 0.75,
179        task_scopes: &[TaskScope::General, TaskScope::ChangeSummary],
180        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
181    },
182    ModeEntry {
183        transform_id: TransformId::DiffCompaction,
184        version: "1.0.0",
185        // Stays --experimental (roadmap.md Phase 5 Task 9, 2026-07-12 re-investigation): the
186        // full-lossy-promotion gate's per_variant breakdown checked the default (body-preserving,
187        // task_scope != ChangeSummary) and header-only (TaskScope::ChangeSummary) forms
188        // separately, as F-013 requires, and BOTH miss the D-005 draft thresholds on their own
189        // (default: quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
190        // 0.5). Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
191        // everything, critical tokens included, when no line matches a unified-diff prefix. See
192        // eval/tasks/FIXTURES.md's "Scorer status" section for the full measured breakdown.
193        conservative_enabled: false,
194        balanced_enabled: false,
195        aggressive_enabled: false,
196        experimental: true,
197        max_ratio_conservative: 0.0,
198        max_ratio_balanced: 0.60,
199        max_ratio_aggressive: 0.70,
200        task_scopes: &[TaskScope::CodeReview, TaskScope::ChangeSummary],
201        applicable_formats: &[
202            InputFormat::PlainText,
203            InputFormat::CommandOutput,
204            InputFormat::GitDiff,
205        ],
206    },
207    // v0.2+ entries (table_compaction, prose_extraction, code_digest, conversation) added
208    // here after their fidelity approval / D-002 scope decisions land.
209];
210
211/// Returns the ordered, applicable transform list for a given (mode, task_scope, format).
212/// `secret_redaction` is not part of this table: the pipeline always runs it first,
213/// unconditionally, before consulting this function.
214pub fn pipeline_for(
215    mode: CompressionMode,
216    task_scope: TaskScope,
217    format: InputFormat,
218    experimental: bool,
219    enabled_ids: &[String],
220    disabled_ids: &[String],
221) -> Vec<&'static ModeEntry> {
222    ALL_ENTRIES
223        .iter()
224        .filter(|e| {
225            let mode_enabled = e.enabled_for(mode);
226            let experimentally_enabled = e.experimental && experimental;
227            let explicitly_enabled = (!e.experimental || experimental)
228                && enabled_ids.iter().any(|id| id == e.transform_id.as_str());
229            mode_enabled || experimentally_enabled || explicitly_enabled
230        })
231        .filter(|e| !disabled_ids.iter().any(|id| id == e.transform_id.as_str()))
232        .filter(|e| e.task_scopes.contains(&TaskScope::All) || e.task_scopes.contains(&task_scope))
233        .filter(|e| e.applies_to_format(format))
234        .collect()
235}
236
237#[cfg(test)]
238mod tests {
239    use super::*;
240
241    #[test]
242    fn transform_id_as_str_matches_canonical_ids() {
243        assert_eq!(TransformId::JsonMinify.as_str(), "json_minify");
244        assert_eq!(TransformId::SchemaCompaction.as_str(), "schema_compaction");
245        assert_eq!(TransformId::LogCompaction.as_str(), "log_compaction");
246        assert_eq!(TransformId::DiffCompaction.as_str(), "diff_compaction");
247    }
248
249    #[test]
250    fn conservative_mode_never_includes_experimental_lossy_transforms() {
251        let entries = pipeline_for(
252            CompressionMode::Conservative,
253            TaskScope::All,
254            InputFormat::PlainText,
255            /* experimental */ true,
256            &[],
257            &[],
258        );
259        assert!(
260            entries
261                .iter()
262                .all(|e| e.transform_id != TransformId::LogCompaction
263                    && e.transform_id != TransformId::DiffCompaction)
264        );
265    }
266
267    #[test]
268    fn balanced_mode_lossless_transforms_apply_to_openai_json() {
269        let entries = pipeline_for(
270            CompressionMode::Balanced,
271            TaskScope::All,
272            InputFormat::OpenAiJson,
273            false,
274            &[],
275            &[],
276        );
277        let ids: Vec<_> = entries.iter().map(|e| e.transform_id).collect();
278        assert!(ids.contains(&TransformId::JsonMinify));
279        assert!(ids.contains(&TransformId::SchemaCompaction));
280    }
281
282    #[test]
283    fn experimental_flag_enables_log_compaction_for_matching_task_scope() {
284        let entries = pipeline_for(
285            CompressionMode::Balanced,
286            TaskScope::General,
287            InputFormat::CommandOutput,
288            true,
289            &[],
290            &[],
291        );
292        assert!(
293            entries
294                .iter()
295                .any(|e| e.transform_id == TransformId::LogCompaction)
296        );
297    }
298
299    #[test]
300    fn log_compaction_skipped_for_non_applicable_format_even_when_experimental() {
301        let entries = pipeline_for(
302            CompressionMode::Balanced,
303            TaskScope::General,
304            InputFormat::OpenAiJson,
305            true,
306            &[],
307            &[],
308        );
309        assert!(
310            !entries
311                .iter()
312                .any(|e| e.transform_id == TransformId::LogCompaction)
313        );
314    }
315
316    #[test]
317    fn disabled_ids_remove_a_transform_even_when_otherwise_enabled() {
318        let entries = pipeline_for(
319            CompressionMode::Balanced,
320            TaskScope::All,
321            InputFormat::OpenAiJson,
322            false,
323            &[],
324            &["json_minify".to_string()],
325        );
326        assert!(
327            !entries
328                .iter()
329                .any(|e| e.transform_id == TransformId::JsonMinify)
330        );
331    }
332
333    #[test]
334    fn diff_compaction_requires_matching_task_scope_even_with_enable_flag() {
335        // enable + experimental together still respect task_scope filtering.
336        let entries = pipeline_for(
337            CompressionMode::Balanced,
338            TaskScope::Debugging,
339            InputFormat::GitDiff,
340            true,
341            &["diff_compaction".to_string()],
342            &[],
343        );
344        assert!(
345            !entries
346                .iter()
347                .any(|e| e.transform_id == TransformId::DiffCompaction)
348        );
349    }
350}
tokenfold_core/modes.rs

tokenfold_core/
modes.rs
