tokenfold_core/
modes.rs

1//! Canonical mode matrix: the single source of truth for which transforms run in which
2//! mode, at what ratio cap, for which task scopes and input formats. `secret_redaction` is
3//! deliberately absent from this table — it runs unconditionally before the pipeline and
4//! cannot be disabled (see `budget::CompressionPolicyBuilder::build`).
5//!
6//! `tests/fixtures/mode_matrix.toml` mirrors this table for cross-surface testing
7//! (INTERFACES.md Part 2 is the authoritative reference for both).
8
9use crate::budget::{CompressionMode, TaskScope};
10use crate::input::InputFormat;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum TransformId {
14    JsonMinify,
15    JsonFieldFold,
16    JsonValueDict,
17    SchemaCompaction,
18    LogCompaction,
19    DiffCompaction,
20}
21
22impl TransformId {
23    pub fn as_str(&self) -> &'static str {
24        match self {
25            TransformId::JsonMinify => "json_minify",
26            TransformId::JsonFieldFold => "json_field_fold",
27            TransformId::JsonValueDict => "json_value_dict",
28            TransformId::SchemaCompaction => "schema_compaction",
29            TransformId::LogCompaction => "log_compaction",
30            TransformId::DiffCompaction => "diff_compaction",
31        }
32    }
33}
34
35#[derive(Debug, Clone, Copy)]
36pub struct ModeEntry {
37    pub transform_id: TransformId,
38    pub version: &'static str,
39    pub conservative_enabled: bool,
40    pub balanced_enabled: bool,
41    pub aggressive_enabled: bool,
42    pub experimental: bool,
43    pub max_ratio_conservative: f64,
44    pub max_ratio_balanced: f64,
45    pub max_ratio_aggressive: f64,
46    pub task_scopes: &'static [TaskScope],
47    pub applicable_formats: &'static [InputFormat],
48}
49
50impl ModeEntry {
51    pub fn enabled_for(&self, mode: CompressionMode) -> bool {
52        match mode {
53            CompressionMode::Conservative => self.conservative_enabled,
54            CompressionMode::Balanced => self.balanced_enabled,
55            CompressionMode::Aggressive => self.aggressive_enabled,
56        }
57    }
58
59    pub fn max_ratio_for(&self, mode: CompressionMode) -> f64 {
60        match mode {
61            CompressionMode::Conservative => self.max_ratio_conservative,
62            CompressionMode::Balanced => self.max_ratio_balanced,
63            CompressionMode::Aggressive => self.max_ratio_aggressive,
64        }
65    }
66
67    fn applies_to_format(&self, format: InputFormat) -> bool {
68        self.applicable_formats.contains(&format)
69    }
70}
71
72// Canonical ordered table — order here IS the pipeline execution order (INTERFACES.md Part 2:
73// lossless before lossy, higher-savings before lower-savings, within each mode).
74//
75// ponytail: `table_compaction` is intentionally omitted. The First Consumer worksheet
76// (plan.md) doesn't name tables among the dominant payload types, so F-019 stays out of
77// scope until a consumer worksheet asks for it (roadmap.md F-019 dependency).
78pub static ALL_ENTRIES: &[ModeEntry] = &[
79    ModeEntry {
80        transform_id: TransformId::JsonMinify,
81        version: "1.0.0",
82        conservative_enabled: true,
83        balanced_enabled: true,
84        aggressive_enabled: true,
85        experimental: false,
86        max_ratio_conservative: 1.0,
87        max_ratio_balanced: 1.0,
88        max_ratio_aggressive: 1.0,
89        task_scopes: &[TaskScope::All],
90        applicable_formats: &[
91            InputFormat::OpenAiJson,
92            InputFormat::AnthropicJson,
93            InputFormat::Json,
94        ],
95    },
96    // json_field_fold (v0.2): reversible columnar fold of arrays of homogeneous objects.
97    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but
98    // it restructures what the model sees, so it stays out of Conservative (same convention
99    // as log_compaction) and only runs on generic Json data, never on OpenAI/Anthropic
100    // message bodies (whose API shape must not change).
101    ModeEntry {
102        transform_id: TransformId::JsonFieldFold,
103        version: "1.0.0",
104        conservative_enabled: false,
105        balanced_enabled: true,
106        aggressive_enabled: true,
107        experimental: false,
108        max_ratio_conservative: 0.0,
109        max_ratio_balanced: 1.0,
110        max_ratio_aggressive: 1.0,
111        task_scopes: &[TaskScope::All],
112        applicable_formats: &[InputFormat::Json],
113    },
114    // json_value_dict (v0.2): reversible value deduplication. Runs AFTER json_field_fold so it
115    // also collapses the repeated nested values folding surfaces across rows. Lossless
116    // (round-trip gated), unrestricted ratio, out of Conservative, generic Json only.
117    ModeEntry {
118        transform_id: TransformId::JsonValueDict,
119        version: "1.0.0",
120        conservative_enabled: false,
121        balanced_enabled: true,
122        aggressive_enabled: true,
123        experimental: false,
124        max_ratio_conservative: 0.0,
125        max_ratio_balanced: 1.0,
126        max_ratio_aggressive: 1.0,
127        task_scopes: &[TaskScope::All],
128        applicable_formats: &[InputFormat::Json],
129    },
130    ModeEntry {
131        transform_id: TransformId::SchemaCompaction,
132        version: "1.0.0",
133        conservative_enabled: true,
134        balanced_enabled: true,
135        aggressive_enabled: true,
136        experimental: false,
137        max_ratio_conservative: 0.15,
138        max_ratio_balanced: 0.30,
139        max_ratio_aggressive: 0.50,
140        task_scopes: &[TaskScope::All],
141        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
142    },
143    ModeEntry {
144        transform_id: TransformId::LogCompaction,
145        version: "1.0.0",
146        // Promoted out of --experimental (roadmap.md Phase 5 Task 9, 2026-07-12): the
147        // full-lossy-promotion fidelity gate clears every D-005 draft threshold cleanly
148        // (quality_retention=1.0, contrastive_failure_rate=0.0, critical_token_survival_rate=1.0).
149        // conservative_enabled stays false — per plan.md's mode table, Conservative never runs
150        // lossy-with-evidence transforms at all, same convention table_compaction documents.
151        conservative_enabled: false,
152        balanced_enabled: true,
153        aggressive_enabled: true,
154        experimental: false,
155        max_ratio_conservative: 0.0,
156        max_ratio_balanced: 0.65, // draft; updated after Phase 2 accuracy@ratio data
157        max_ratio_aggressive: 0.75,
158        task_scopes: &[TaskScope::General, TaskScope::ChangeSummary],
159        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
160    },
161    ModeEntry {
162        transform_id: TransformId::DiffCompaction,
163        version: "1.0.0",
164        // Stays --experimental (roadmap.md Phase 5 Task 9, 2026-07-12 re-investigation): the
165        // full-lossy-promotion gate's per_variant breakdown checked the default (body-preserving,
166        // task_scope != ChangeSummary) and header-only (TaskScope::ChangeSummary) forms
167        // separately, as F-013 requires, and BOTH miss the D-005 draft thresholds on their own
168        // (default: quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
169        // 0.5). Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
170        // everything, critical tokens included, when no line matches a unified-diff prefix. See
171        // eval/tasks/FIXTURES.md's "Scorer status" section for the full measured breakdown.
172        conservative_enabled: false,
173        balanced_enabled: false,
174        aggressive_enabled: false,
175        experimental: true,
176        max_ratio_conservative: 0.0,
177        max_ratio_balanced: 0.60,
178        max_ratio_aggressive: 0.70,
179        task_scopes: &[TaskScope::CodeReview, TaskScope::ChangeSummary],
180        applicable_formats: &[
181            InputFormat::PlainText,
182            InputFormat::CommandOutput,
183            InputFormat::GitDiff,
184        ],
185    },
186    // v0.2+ entries (table_compaction, prose_extraction, code_digest, conversation) added
187    // here after their fidelity approval / D-002 scope decisions land.
188];
189
190/// Returns the ordered, applicable transform list for a given (mode, task_scope, format).
191/// `secret_redaction` is not part of this table: the pipeline always runs it first,
192/// unconditionally, before consulting this function.
193pub fn pipeline_for(
194    mode: CompressionMode,
195    task_scope: TaskScope,
196    format: InputFormat,
197    experimental: bool,
198    enabled_ids: &[String],
199    disabled_ids: &[String],
200) -> Vec<&'static ModeEntry> {
201    ALL_ENTRIES
202        .iter()
203        .filter(|e| {
204            let mode_enabled = e.enabled_for(mode);
205            let experimentally_enabled = e.experimental && experimental;
206            let explicitly_enabled = (!e.experimental || experimental)
207                && enabled_ids.iter().any(|id| id == e.transform_id.as_str());
208            mode_enabled || experimentally_enabled || explicitly_enabled
209        })
210        .filter(|e| !disabled_ids.iter().any(|id| id == e.transform_id.as_str()))
211        .filter(|e| e.task_scopes.contains(&TaskScope::All) || e.task_scopes.contains(&task_scope))
212        .filter(|e| e.applies_to_format(format))
213        .collect()
214}
215
216#[cfg(test)]
217mod tests {
218    use super::*;
219
220    #[test]
221    fn transform_id_as_str_matches_canonical_ids() {
222        assert_eq!(TransformId::JsonMinify.as_str(), "json_minify");
223        assert_eq!(TransformId::SchemaCompaction.as_str(), "schema_compaction");
224        assert_eq!(TransformId::LogCompaction.as_str(), "log_compaction");
225        assert_eq!(TransformId::DiffCompaction.as_str(), "diff_compaction");
226    }
227
228    #[test]
229    fn conservative_mode_never_includes_experimental_lossy_transforms() {
230        let entries = pipeline_for(
231            CompressionMode::Conservative,
232            TaskScope::All,
233            InputFormat::PlainText,
234            /* experimental */ true,
235            &[],
236            &[],
237        );
238        assert!(
239            entries
240                .iter()
241                .all(|e| e.transform_id != TransformId::LogCompaction
242                    && e.transform_id != TransformId::DiffCompaction)
243        );
244    }
245
246    #[test]
247    fn balanced_mode_lossless_transforms_apply_to_openai_json() {
248        let entries = pipeline_for(
249            CompressionMode::Balanced,
250            TaskScope::All,
251            InputFormat::OpenAiJson,
252            false,
253            &[],
254            &[],
255        );
256        let ids: Vec<_> = entries.iter().map(|e| e.transform_id).collect();
257        assert!(ids.contains(&TransformId::JsonMinify));
258        assert!(ids.contains(&TransformId::SchemaCompaction));
259    }
260
261    #[test]
262    fn experimental_flag_enables_log_compaction_for_matching_task_scope() {
263        let entries = pipeline_for(
264            CompressionMode::Balanced,
265            TaskScope::General,
266            InputFormat::CommandOutput,
267            true,
268            &[],
269            &[],
270        );
271        assert!(
272            entries
273                .iter()
274                .any(|e| e.transform_id == TransformId::LogCompaction)
275        );
276    }
277
278    #[test]
279    fn log_compaction_skipped_for_non_applicable_format_even_when_experimental() {
280        let entries = pipeline_for(
281            CompressionMode::Balanced,
282            TaskScope::General,
283            InputFormat::OpenAiJson,
284            true,
285            &[],
286            &[],
287        );
288        assert!(
289            !entries
290                .iter()
291                .any(|e| e.transform_id == TransformId::LogCompaction)
292        );
293    }
294
295    #[test]
296    fn disabled_ids_remove_a_transform_even_when_otherwise_enabled() {
297        let entries = pipeline_for(
298            CompressionMode::Balanced,
299            TaskScope::All,
300            InputFormat::OpenAiJson,
301            false,
302            &[],
303            &["json_minify".to_string()],
304        );
305        assert!(
306            !entries
307                .iter()
308                .any(|e| e.transform_id == TransformId::JsonMinify)
309        );
310    }
311
312    #[test]
313    fn diff_compaction_requires_matching_task_scope_even_with_enable_flag() {
314        // enable + experimental together still respect task_scope filtering.
315        let entries = pipeline_for(
316            CompressionMode::Balanced,
317            TaskScope::Debugging,
318            InputFormat::GitDiff,
319            true,
320            &["diff_compaction".to_string()],
321            &[],
322        );
323        assert!(
324            !entries
325                .iter()
326                .any(|e| e.transform_id == TransformId::DiffCompaction)
327        );
328    }
329}
tokenfold_core/modes.rs

tokenfold_core/
modes.rs
