Skip to main content

code_system_graph_core/
batch.rs

1use code_system_graph_model::ArtifactFingerprint;
2use serde::Serialize;
3use serde::de::DeserializeOwned;
4use thiserror::Error;
5
6use crate::{
7    EXTRACTION_CONTRACT_VERSION, ExtractionBudgets, ExtractionLimitExceeded, ExtractionResource, ExtractionTracker
8};
9
10/// Complete transient output owned by one extractor input.
11#[derive(Debug, Clone, PartialEq, Eq)]
12pub struct ExtractorBatch<T> {
13    /// Fingerprint that identifies and invalidates this batch.
14    pub source: ArtifactFingerprint,
15    /// Deterministically ordered extracted outputs.
16    pub outputs: Vec<T>,
17}
18
19impl<T> ExtractorBatch<T> {
20    /// Creates one source-owned extractor batch.
21    #[must_use]
22    pub fn new(source: ArtifactFingerprint, outputs: Vec<T>) -> Self {
23        Self { source, outputs }
24    }
25}
26
27/// Error returned while encoding or decoding a source-owned extractor batch.
28#[derive(Debug, Error, PartialEq, Eq)]
29pub enum BatchPlanError {
30    /// A source-owned output payload could not be serialized or decoded.
31    #[error("invalid extractor batch payload: {0}")]
32    InvalidPayload(String),
33    /// One per-invocation extraction resource exceeded its configured maximum.
34    #[error(transparent)]
35    ExtractionLimit(#[from] ExtractionLimitExceeded),
36    /// An output count cannot be represented by the persistence model.
37    #[error("extractor batch output count exceeds the supported range")]
38    OutputCountOverflow,
39    /// Persisted output count does not match the decoded payload.
40    #[error("extractor batch output count mismatch: stored {stored}, decoded {decoded}")]
41    OutputCountMismatch {
42        /// Count stored alongside the payload.
43        stored: u64,
44        /// Number of decoded outputs.
45        decoded: usize,
46    },
47}
48
49/// Encodes one typed extractor batch for atomic snapshot persistence.
50///
51/// # Errors
52///
53/// Returns [`BatchPlanError`] when outputs cannot be encoded or their count exceeds `u64`.
54pub fn store_extractor_batch<T: Serialize>(
55    batch: &ExtractorBatch<T>,
56    tracker: &mut ExtractionTracker,
57    source_was_lossy: bool,
58) -> Result<code_system_graph_model::StoredExtractorBatch, BatchPlanError> {
59    tracker.ensure_observations(u64::try_from(batch.outputs.len()).unwrap_or(u64::MAX))?;
60    let mut writer = tracker.bounded_json_writer();
61    if let Err(error) = serde_json::to_writer(&mut writer, &batch.outputs) {
62        if let Some(limit) = tracker.output_limit_error(&writer) {
63            return Err(limit.into());
64        }
65        return Err(BatchPlanError::InvalidPayload(error.to_string()));
66    }
67    Ok(code_system_graph_model::StoredExtractorBatch {
68        source: batch.source.clone(),
69        extractor_version: EXTRACTION_CONTRACT_VERSION.to_owned(),
70        budget_fingerprint: tracker.budgets().fingerprint(),
71        source_was_lossy,
72        output_count: u64::try_from(batch.outputs.len())
73            .map_err(|_| BatchPlanError::OutputCountOverflow)?,
74        payload: writer.into_inner(),
75    })
76}
77
78/// Decodes one persisted source-owned output batch.
79///
80/// # Errors
81///
82/// Returns [`BatchPlanError`] when the payload schema is invalid or its count is inconsistent.
83pub fn load_extractor_batch<T: DeserializeOwned>(
84    stored: &code_system_graph_model::StoredExtractorBatch,
85) -> Result<ExtractorBatch<T>, BatchPlanError> {
86    load_extractor_batch_with_budgets(stored, &ExtractionBudgets::default())
87}
88
89/// Decodes one persisted source-owned output batch after enforcing effective budgets.
90///
91/// The count and byte checks deliberately precede deserialization so a corrupt or untrusted
92/// persisted batch cannot force allocations beyond the active extraction policy.
93///
94/// # Errors
95///
96/// Returns [`BatchPlanError`] when the payload exceeds `budgets`, its schema is invalid, or its
97/// count is inconsistent.
98pub fn load_extractor_batch_with_budgets<T: DeserializeOwned>(
99    stored: &code_system_graph_model::StoredExtractorBatch,
100    budgets: &ExtractionBudgets,
101) -> Result<ExtractorBatch<T>, BatchPlanError> {
102    if stored.output_count > budgets.max_observations_per_artifact {
103        return Err(ExtractionLimitExceeded {
104            artifact: stored.source.path.display.clone(),
105            extractor: stored.source.extractor.clone(),
106            resource: ExtractionResource::Observations,
107            observed: stored.output_count,
108            maximum: budgets.max_observations_per_artifact,
109        }
110        .into());
111    }
112    let observed = u64::try_from(stored.payload.len()).unwrap_or(u64::MAX);
113    if observed > budgets.max_serialized_output_bytes_per_artifact {
114        return Err(ExtractionLimitExceeded {
115            artifact: stored.source.path.display.clone(),
116            extractor: stored.source.extractor.clone(),
117            resource: ExtractionResource::SerializedOutputBytes,
118            observed,
119            maximum: budgets.max_serialized_output_bytes_per_artifact,
120        }
121        .into());
122    }
123    let outputs: Vec<T> = serde_json::from_slice(&stored.payload)
124        .map_err(|error| BatchPlanError::InvalidPayload(error.to_string()))?;
125    if usize::try_from(stored.output_count).ok() != Some(outputs.len()) {
126        return Err(BatchPlanError::OutputCountMismatch {
127            stored: stored.output_count,
128            decoded: outputs.len(),
129        });
130    }
131    Ok(ExtractorBatch::new(stored.source.clone(), outputs))
132}
133
134#[cfg(test)]
135mod tests {
136    use code_system_graph_model::{
137        ArtifactFingerprint, CheckoutId, NativePath, NativePathEncoding, RepoId
138    };
139
140    use super::{
141        BatchPlanError, ExtractorBatch, load_extractor_batch, load_extractor_batch_with_budgets, store_extractor_batch
142    };
143    use crate::{ExtractionBudgets, ExtractionLimitExceeded, ExtractionTracker};
144
145    fn tracker() -> ExtractionTracker {
146        ExtractionTracker::new("src/routes.rs", "test", &ExtractionBudgets::default())
147    }
148
149    fn path(value: &str) -> NativePath {
150        NativePath {
151            encoding: NativePathEncoding::Utf8,
152            bytes: value.as_bytes().to_vec(),
153            display: value.to_owned(),
154        }
155    }
156
157    fn batch(source: &str, hash: &str, outputs: &[&str]) -> ExtractorBatch<String> {
158        ExtractorBatch::new(
159            ArtifactFingerprint {
160                repo_id: RepoId::new("repo:api"),
161                checkout_id: CheckoutId::new("checkout:api"),
162                path: path(source),
163                extractor: "code-system-graph.http.openapi".to_owned(),
164                content_hash: hash.to_owned(),
165                size_bytes: 1,
166            },
167            outputs.iter().map(|output| (*output).to_owned()).collect(),
168        )
169    }
170
171    #[test]
172    fn stored_batch_should_round_trip_without_source_text() {
173        let original = batch("src/routes.rs", "hash", &["GET:/orders", "POST:/orders"]);
174        let result = store_extractor_batch(&original, &mut tracker(), false)
175            .and_then(|stored| load_extractor_batch::<String>(&stored));
176
177        assert_eq!(result, Ok(original));
178    }
179
180    #[test]
181    fn stored_batch_should_reject_inconsistent_output_count() {
182        let original = batch("src/routes.rs", "hash", &["GET:/orders"]);
183        let result =
184            store_extractor_batch(&original, &mut tracker(), false).and_then(|mut stored| {
185                stored.output_count = 2;
186                load_extractor_batch::<String>(&stored)
187            });
188
189        assert!(matches!(
190            result,
191            Err(BatchPlanError::OutputCountMismatch {
192                stored: 2,
193                decoded: 1
194            })
195        ));
196    }
197
198    #[test]
199    fn stored_batch_should_check_payload_limit_before_decoding() {
200        let original = batch("src/routes.rs", "hash", &["GET:/orders"]);
201        let stored = store_extractor_batch(&original, &mut tracker(), false).expect("stored batch");
202        let exact = u64::try_from(stored.payload.len()).expect("payload length");
203        let exact_budgets = ExtractionBudgets {
204            max_serialized_output_bytes_per_artifact: exact,
205            ..ExtractionBudgets::default()
206        };
207        let below_budgets = ExtractionBudgets {
208            max_serialized_output_bytes_per_artifact: exact - 1,
209            ..ExtractionBudgets::default()
210        };
211
212        assert_eq!(
213            load_extractor_batch_with_budgets::<String>(&stored, &exact_budgets),
214            Ok(original)
215        );
216        assert!(matches!(
217            load_extractor_batch_with_budgets::<String>(&stored, &below_budgets),
218            Err(BatchPlanError::ExtractionLimit(ExtractionLimitExceeded {
219                resource: crate::ExtractionResource::SerializedOutputBytes,
220                observed,
221                maximum,
222                ..
223            })) if observed == exact && maximum == exact - 1
224        ));
225    }
226
227    #[test]
228    fn stored_batch_should_check_observation_limit_before_decoding() {
229        let original = batch("src/routes.rs", "hash", &["GET:/orders", "POST:/orders"]);
230        let stored = store_extractor_batch(&original, &mut tracker(), false).expect("stored batch");
231        let budgets = ExtractionBudgets {
232            max_observations_per_artifact: 1,
233            ..ExtractionBudgets::default()
234        };
235
236        assert!(matches!(
237            load_extractor_batch_with_budgets::<String>(&stored, &budgets),
238            Err(BatchPlanError::ExtractionLimit(ExtractionLimitExceeded {
239                resource: crate::ExtractionResource::Observations,
240                observed: 2,
241                maximum: 1,
242                ..
243            }))
244        ));
245    }
246}