Skip to main content

tokencompact_builtins/
lib.rs

1//! Deterministic, source-agnostic reduction of compiler, test, and tool text.
2//!
3//! The crate performs no I/O and has no async, provider protocol, storage, or
4//! runner dependency. Callers provide already acquired text and an explicit
5//! redaction policy; only normalized and redacted diagnostics are returned.
6
7mod diagnostics;
8mod finalize;
9mod model;
10mod redaction;
11mod streaming;
12mod test_failures;
13mod test_log;
14mod text;
15
16pub use diagnostics::{
17    JavaScriptTestDiagnosticParser, JavaTestDiagnosticParser, PythonDiagnosticParser,
18    parse_go_diagnostic,
19};
20pub use model::{
21    Budget, Diagnostic, DiagnosticClass, EndReason, EvidenceQuality, FallbackPolicy, Limits,
22    Location, LogLine, Provenance, Reduction, ReductionOptions, ReductionStats, Scope, ScopeKind,
23    SessionOptions, Severity, Stream, TestFailure, TextInput,
24};
25pub use redaction::{NoPathMapping, NoRedaction, PathMapper, Redactor};
26pub use streaming::{BuiltinDiagnosticParser, BuiltinParserOptions, builtin_parser_plan};
27pub use test_failures::{TestFailureAccumulator, TestFailureEvidence};
28pub use test_log::{TestLogReducer, TestLogReduction};
29pub use text::{deduplicate_lines, normalize_terminal_text};
30pub use tokencompact_core::{
31    Emitter, GenericRanker, OutputPolicy, Parser, ParserPlan, ParserPlanError, RankKey, Ranker,
32    ReductionSession, ScopeBoundary,
33};
34
35/// Reduces one or more text inputs with deterministic parser and input order.
36#[must_use]
37pub fn reduce(
38    inputs: &[TextInput<'_>],
39    options: &ReductionOptions,
40    redactor: &dyn Redactor,
41) -> Reduction {
42    reduce_with_policy(
43        inputs,
44        options,
45        OutputPolicy::new(redactor, &NoPathMapping, &GenericRanker),
46    )
47}
48
49/// Reduces batch inputs with caller-owned path mapping, redaction, and ranking.
50#[must_use]
51pub fn reduce_with_policy(
52    inputs: &[TextInput<'_>],
53    options: &ReductionOptions,
54    policy: OutputPolicy<'_>,
55) -> Reduction {
56    let mut diagnostics = Vec::new();
57    for input in inputs {
58        let start = diagnostics.len();
59        diagnostics::add_text_diagnostics(input.text, &mut diagnostics, options.fallback);
60        for diagnostic in &mut diagnostics[start..] {
61            diagnostic.provenance = input.provenance.cloned();
62            if diagnostic.location.is_some() && diagnostic.quality == EvidenceQuality::Structured {
63                diagnostic.quality = EvidenceQuality::Located;
64            }
65        }
66    }
67    finalize::finalize_with_policy(diagnostics, options.budget, policy)
68}
69
70#[cfg(feature = "test-support")]
71#[doc(hidden)]
72pub fn __parse_cpp_diagnostic(input: &str) -> Option<Diagnostic> {
73    diagnostics::parse_cpp_diagnostic(input)
74}
75
76#[cfg(feature = "test-support")]
77#[doc(hidden)]
78pub fn __parse_cpp_linker_diagnostic(input: &str) -> Option<Diagnostic> {
79    diagnostics::parse_cpp_linker_diagnostic(input)
80}
81
82#[cfg(feature = "test-support")]
83#[doc(hidden)]
84pub fn __cpp_path_end(input: &str, delimiter: char) -> Option<usize> {
85    diagnostics::cpp_path_end(input, delimiter)
86}
87
88#[cfg(feature = "test-support")]
89#[doc(hidden)]
90pub fn __parse_swc_diagnostics(input: &str) -> Vec<Diagnostic> {
91    diagnostics::parse_swc_diagnostics(input)
92}
93
94#[cfg(feature = "test-support")]
95#[doc(hidden)]
96pub fn __parse_protobuf_diagnostic(input: &str) -> Option<Diagnostic> {
97    diagnostics::parse_protobuf_diagnostic(input)
98}
99
100#[cfg(feature = "test-support")]
101#[doc(hidden)]
102pub fn __parse_typescript_diagnostic(input: &str) -> Option<Diagnostic> {
103    diagnostics::parse_typescript_diagnostic(input)
104}
105
106#[cfg(test)]
107mod tests {
108    use super::*;
109
110    #[test]
111    fn reduces_compiler_and_test_logs_without_provider_objects() {
112        let compiler = Provenance::new("compile").with_label("rust");
113        let test = Provenance::new("test").with_label("node");
114        let inputs = [
115            TextInput::new(
116                b"error[E0308]: mismatched types\n --> src/lib.rs:7:5\n  |\n7 | value\n  | ^ expected u32, found &str",
117            )
118            .with_provenance(&compiler),
119            TextInput::new(b"TypeError: total is not a function\n    at src/invoice.test.js:8:3")
120                .with_provenance(&test),
121        ];
122        let reduction = reduce(
123            &inputs,
124            &ReductionOptions {
125                budget: Budget::unbounded(),
126                ..ReductionOptions::default()
127            },
128            &NoRedaction,
129        );
130
131        assert_eq!(reduction.diagnostics.len(), 2);
132        assert_eq!(reduction.diagnostics[0].class, DiagnosticClass::Compiler);
133        assert_eq!(
134            reduction.diagnostics[0].location.as_ref().unwrap().path,
135            "src/lib.rs"
136        );
137        assert_eq!(reduction.diagnostics[1].class, DiagnosticClass::Test);
138        assert_eq!(
139            reduction.diagnostics[1].provenance.as_ref().unwrap().source,
140            "test"
141        );
142    }
143
144    #[test]
145    fn provenance_keeps_otherwise_identical_inputs_distinct() {
146        let first = Provenance::new("stderr");
147        let second = Provenance::new("stdout");
148        let inputs = [
149            TextInput::new(b"src/main.go:4:2: undefined: value").with_provenance(&first),
150            TextInput::new(b"src/main.go:4:2: undefined: value").with_provenance(&second),
151        ];
152        let reduction = reduce(
153            &inputs,
154            &ReductionOptions {
155                budget: Budget::unbounded(),
156                ..ReductionOptions::default()
157            },
158            &NoRedaction,
159        );
160        assert_eq!(reduction.diagnostics.len(), 2);
161    }
162
163    #[test]
164    fn redacts_messages_locations_and_provenance_before_return() {
165        let provenance = Provenance::new("job-token=source").with_label("token=label");
166        let inputs = [
167            TextInput::new(b"token=path/main.go:4:2: token=message").with_provenance(&provenance)
168        ];
169        let redact = |value: &str| value.replace("token=", "[REDACTED]-");
170        let reduction = reduce(
171            &inputs,
172            &ReductionOptions {
173                budget: Budget::unbounded(),
174                ..ReductionOptions::default()
175            },
176            &redact,
177        );
178        let encoded = serde_json::to_string(&reduction).unwrap();
179        assert!(!encoded.contains("token="));
180        assert!(encoded.contains("[REDACTED]-path/main.go"));
181        assert!(encoded.contains("[REDACTED]-message"));
182        assert!(encoded.contains("job-[REDACTED]-source"));
183    }
184
185    #[test]
186    fn fallback_can_be_disabled_without_disabling_structured_parsers() {
187        let inputs = [TextInput::new(
188            b"generic fatal: wrapper\nsrc/main.go:4:2: undefined: value",
189        )];
190        let reduction = reduce(
191            &inputs,
192            &ReductionOptions {
193                budget: Budget::unbounded(),
194                fallback: FallbackPolicy::Disabled,
195            },
196            &NoRedaction,
197        );
198        assert_eq!(reduction.diagnostics.len(), 1);
199        assert_eq!(reduction.diagnostics[0].message, "undefined: value");
200    }
201}