tokenfold_core/
modes.rs

1//! Canonical mode matrix: the single source of truth for which transforms run in which
2//! mode, at what ratio cap, for which task scopes and input formats. `secret_redaction` is
3//! deliberately absent from this table — it runs unconditionally before the pipeline and
4//! cannot be disabled (see `budget::CompressionPolicyBuilder::build`).
5//!
6//! `tests/fixtures/mode_matrix.toml` mirrors this table for cross-surface testing
7//! (INTERFACES.md Part 2 is the authoritative reference for both).
8
9use crate::budget::{CompressionMode, TaskScope};
10use crate::input::InputFormat;
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum TransformId {
14    JsonMinify,
15    SchemaCompaction,
16    LogCompaction,
17    DiffCompaction,
18}
19
20impl TransformId {
21    pub fn as_str(&self) -> &'static str {
22        match self {
23            TransformId::JsonMinify => "json_minify",
24            TransformId::SchemaCompaction => "schema_compaction",
25            TransformId::LogCompaction => "log_compaction",
26            TransformId::DiffCompaction => "diff_compaction",
27        }
28    }
29}
30
31#[derive(Debug, Clone, Copy)]
32pub struct ModeEntry {
33    pub transform_id: TransformId,
34    pub version: &'static str,
35    pub conservative_enabled: bool,
36    pub balanced_enabled: bool,
37    pub aggressive_enabled: bool,
38    pub experimental: bool,
39    pub max_ratio_conservative: f64,
40    pub max_ratio_balanced: f64,
41    pub max_ratio_aggressive: f64,
42    pub task_scopes: &'static [TaskScope],
43    pub applicable_formats: &'static [InputFormat],
44}
45
46impl ModeEntry {
47    pub fn enabled_for(&self, mode: CompressionMode) -> bool {
48        match mode {
49            CompressionMode::Conservative => self.conservative_enabled,
50            CompressionMode::Balanced => self.balanced_enabled,
51            CompressionMode::Aggressive => self.aggressive_enabled,
52        }
53    }
54
55    pub fn max_ratio_for(&self, mode: CompressionMode) -> f64 {
56        match mode {
57            CompressionMode::Conservative => self.max_ratio_conservative,
58            CompressionMode::Balanced => self.max_ratio_balanced,
59            CompressionMode::Aggressive => self.max_ratio_aggressive,
60        }
61    }
62
63    fn applies_to_format(&self, format: InputFormat) -> bool {
64        self.applicable_formats.contains(&format)
65    }
66}
67
68// Canonical ordered table — order here IS the pipeline execution order (INTERFACES.md Part 2:
69// lossless before lossy, higher-savings before lower-savings, within each mode).
70//
71// ponytail: `table_compaction` is intentionally omitted. The First Consumer worksheet
72// (plan.md) doesn't name tables among the dominant payload types, so F-019 stays out of
73// scope until a consumer worksheet asks for it (roadmap.md F-019 dependency).
74pub static ALL_ENTRIES: &[ModeEntry] = &[
75    ModeEntry {
76        transform_id: TransformId::JsonMinify,
77        version: "1.0.0",
78        conservative_enabled: true,
79        balanced_enabled: true,
80        aggressive_enabled: true,
81        experimental: false,
82        max_ratio_conservative: 1.0,
83        max_ratio_balanced: 1.0,
84        max_ratio_aggressive: 1.0,
85        task_scopes: &[TaskScope::All],
86        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
87    },
88    ModeEntry {
89        transform_id: TransformId::SchemaCompaction,
90        version: "1.0.0",
91        conservative_enabled: true,
92        balanced_enabled: true,
93        aggressive_enabled: true,
94        experimental: false,
95        max_ratio_conservative: 0.15,
96        max_ratio_balanced: 0.30,
97        max_ratio_aggressive: 0.50,
98        task_scopes: &[TaskScope::All],
99        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
100    },
101    ModeEntry {
102        transform_id: TransformId::LogCompaction,
103        version: "1.0.0",
104        // Promoted out of --experimental (roadmap.md Phase 5 Task 9, 2026-07-12): the
105        // full-lossy-promotion fidelity gate clears every D-005 draft threshold cleanly
106        // (quality_retention=1.0, contrastive_failure_rate=0.0, critical_token_survival_rate=1.0).
107        // conservative_enabled stays false — per plan.md's mode table, Conservative never runs
108        // lossy-with-evidence transforms at all, same convention table_compaction documents.
109        conservative_enabled: false,
110        balanced_enabled: true,
111        aggressive_enabled: true,
112        experimental: false,
113        max_ratio_conservative: 0.0,
114        max_ratio_balanced: 0.65, // draft; updated after Phase 2 accuracy@ratio data
115        max_ratio_aggressive: 0.75,
116        task_scopes: &[TaskScope::General, TaskScope::ChangeSummary],
117        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
118    },
119    ModeEntry {
120        transform_id: TransformId::DiffCompaction,
121        version: "1.0.0",
122        // Stays --experimental (roadmap.md Phase 5 Task 9, 2026-07-12 re-investigation): the
123        // full-lossy-promotion gate's per_variant breakdown checked the default (body-preserving,
124        // task_scope != ChangeSummary) and header-only (TaskScope::ChangeSummary) forms
125        // separately, as F-013 requires, and BOTH miss the D-005 draft thresholds on their own
126        // (default: quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
127        // 0.5). Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
128        // everything, critical tokens included, when no line matches a unified-diff prefix. See
129        // eval/tasks/FIXTURES.md's "Scorer status" section for the full measured breakdown.
130        conservative_enabled: false,
131        balanced_enabled: false,
132        aggressive_enabled: false,
133        experimental: true,
134        max_ratio_conservative: 0.0,
135        max_ratio_balanced: 0.60,
136        max_ratio_aggressive: 0.70,
137        task_scopes: &[TaskScope::CodeReview, TaskScope::ChangeSummary],
138        applicable_formats: &[
139            InputFormat::PlainText,
140            InputFormat::CommandOutput,
141            InputFormat::GitDiff,
142        ],
143    },
144    // v0.2+ entries (table_compaction, prose_extraction, code_digest, conversation) added
145    // here after their fidelity approval / D-002 scope decisions land.
146];
147
148/// Returns the ordered, applicable transform list for a given (mode, task_scope, format).
149/// `secret_redaction` is not part of this table: the pipeline always runs it first,
150/// unconditionally, before consulting this function.
151pub fn pipeline_for(
152    mode: CompressionMode,
153    task_scope: TaskScope,
154    format: InputFormat,
155    experimental: bool,
156    enabled_ids: &[String],
157    disabled_ids: &[String],
158) -> Vec<&'static ModeEntry> {
159    ALL_ENTRIES
160        .iter()
161        .filter(|e| {
162            let mode_enabled = e.enabled_for(mode);
163            let experimentally_enabled = e.experimental && experimental;
164            let explicitly_enabled = (!e.experimental || experimental)
165                && enabled_ids.iter().any(|id| id == e.transform_id.as_str());
166            mode_enabled || experimentally_enabled || explicitly_enabled
167        })
168        .filter(|e| !disabled_ids.iter().any(|id| id == e.transform_id.as_str()))
169        .filter(|e| e.task_scopes.contains(&TaskScope::All) || e.task_scopes.contains(&task_scope))
170        .filter(|e| e.applies_to_format(format))
171        .collect()
172}
173
174#[cfg(test)]
175mod tests {
176    use super::*;
177
178    #[test]
179    fn transform_id_as_str_matches_canonical_ids() {
180        assert_eq!(TransformId::JsonMinify.as_str(), "json_minify");
181        assert_eq!(TransformId::SchemaCompaction.as_str(), "schema_compaction");
182        assert_eq!(TransformId::LogCompaction.as_str(), "log_compaction");
183        assert_eq!(TransformId::DiffCompaction.as_str(), "diff_compaction");
184    }
185
186    #[test]
187    fn conservative_mode_never_includes_experimental_lossy_transforms() {
188        let entries = pipeline_for(
189            CompressionMode::Conservative,
190            TaskScope::All,
191            InputFormat::PlainText,
192            /* experimental */ true,
193            &[],
194            &[],
195        );
196        assert!(
197            entries
198                .iter()
199                .all(|e| e.transform_id != TransformId::LogCompaction
200                    && e.transform_id != TransformId::DiffCompaction)
201        );
202    }
203
204    #[test]
205    fn balanced_mode_lossless_transforms_apply_to_openai_json() {
206        let entries = pipeline_for(
207            CompressionMode::Balanced,
208            TaskScope::All,
209            InputFormat::OpenAiJson,
210            false,
211            &[],
212            &[],
213        );
214        let ids: Vec<_> = entries.iter().map(|e| e.transform_id).collect();
215        assert!(ids.contains(&TransformId::JsonMinify));
216        assert!(ids.contains(&TransformId::SchemaCompaction));
217    }
218
219    #[test]
220    fn experimental_flag_enables_log_compaction_for_matching_task_scope() {
221        let entries = pipeline_for(
222            CompressionMode::Balanced,
223            TaskScope::General,
224            InputFormat::CommandOutput,
225            true,
226            &[],
227            &[],
228        );
229        assert!(
230            entries
231                .iter()
232                .any(|e| e.transform_id == TransformId::LogCompaction)
233        );
234    }
235
236    #[test]
237    fn log_compaction_skipped_for_non_applicable_format_even_when_experimental() {
238        let entries = pipeline_for(
239            CompressionMode::Balanced,
240            TaskScope::General,
241            InputFormat::OpenAiJson,
242            true,
243            &[],
244            &[],
245        );
246        assert!(
247            !entries
248                .iter()
249                .any(|e| e.transform_id == TransformId::LogCompaction)
250        );
251    }
252
253    #[test]
254    fn disabled_ids_remove_a_transform_even_when_otherwise_enabled() {
255        let entries = pipeline_for(
256            CompressionMode::Balanced,
257            TaskScope::All,
258            InputFormat::OpenAiJson,
259            false,
260            &[],
261            &["json_minify".to_string()],
262        );
263        assert!(
264            !entries
265                .iter()
266                .any(|e| e.transform_id == TransformId::JsonMinify)
267        );
268    }
269
270    #[test]
271    fn diff_compaction_requires_matching_task_scope_even_with_enable_flag() {
272        // enable + experimental together still respect task_scope filtering.
273        let entries = pipeline_for(
274            CompressionMode::Balanced,
275            TaskScope::Debugging,
276            InputFormat::GitDiff,
277            true,
278            &["diff_compaction".to_string()],
279            &[],
280        );
281        assert!(
282            !entries
283                .iter()
284                .any(|e| e.transform_id == TransformId::DiffCompaction)
285        );
286    }
287}
tokenfold_core/modes.rs

tokenfold_core/
modes.rs
