1use code_system_graph_model::ArtifactFingerprint;
2use serde::Serialize;
3use serde::de::DeserializeOwned;
4use thiserror::Error;
5
6use crate::{
7 EXTRACTION_CONTRACT_VERSION, ExtractionBudgets, ExtractionLimitExceeded, ExtractionResource, ExtractionTracker
8};
9
10#[derive(Debug, Clone, PartialEq, Eq)]
12pub struct ExtractorBatch<T> {
13 pub source: ArtifactFingerprint,
15 pub outputs: Vec<T>,
17}
18
19impl<T> ExtractorBatch<T> {
20 #[must_use]
22 pub fn new(source: ArtifactFingerprint, outputs: Vec<T>) -> Self {
23 Self { source, outputs }
24 }
25}
26
27#[derive(Debug, Error, PartialEq, Eq)]
29pub enum BatchPlanError {
30 #[error("invalid extractor batch payload: {0}")]
32 InvalidPayload(String),
33 #[error(transparent)]
35 ExtractionLimit(#[from] ExtractionLimitExceeded),
36 #[error("extractor batch output count exceeds the supported range")]
38 OutputCountOverflow,
39 #[error("extractor batch output count mismatch: stored {stored}, decoded {decoded}")]
41 OutputCountMismatch {
42 stored: u64,
44 decoded: usize,
46 },
47}
48
49pub fn store_extractor_batch<T: Serialize>(
55 batch: &ExtractorBatch<T>,
56 tracker: &mut ExtractionTracker,
57 source_was_lossy: bool,
58) -> Result<code_system_graph_model::StoredExtractorBatch, BatchPlanError> {
59 tracker.ensure_observations(u64::try_from(batch.outputs.len()).unwrap_or(u64::MAX))?;
60 let mut writer = tracker.bounded_json_writer();
61 if let Err(error) = serde_json::to_writer(&mut writer, &batch.outputs) {
62 if let Some(limit) = tracker.output_limit_error(&writer) {
63 return Err(limit.into());
64 }
65 return Err(BatchPlanError::InvalidPayload(error.to_string()));
66 }
67 Ok(code_system_graph_model::StoredExtractorBatch {
68 source: batch.source.clone(),
69 extractor_version: EXTRACTION_CONTRACT_VERSION.to_owned(),
70 budget_fingerprint: tracker.budgets().fingerprint(),
71 source_was_lossy,
72 output_count: u64::try_from(batch.outputs.len())
73 .map_err(|_| BatchPlanError::OutputCountOverflow)?,
74 payload: writer.into_inner(),
75 })
76}
77
78pub fn load_extractor_batch<T: DeserializeOwned>(
84 stored: &code_system_graph_model::StoredExtractorBatch,
85) -> Result<ExtractorBatch<T>, BatchPlanError> {
86 load_extractor_batch_with_budgets(stored, &ExtractionBudgets::default())
87}
88
89pub fn load_extractor_batch_with_budgets<T: DeserializeOwned>(
99 stored: &code_system_graph_model::StoredExtractorBatch,
100 budgets: &ExtractionBudgets,
101) -> Result<ExtractorBatch<T>, BatchPlanError> {
102 if stored.output_count > budgets.max_observations_per_artifact {
103 return Err(ExtractionLimitExceeded {
104 artifact: stored.source.path.display.clone(),
105 extractor: stored.source.extractor.clone(),
106 resource: ExtractionResource::Observations,
107 observed: stored.output_count,
108 maximum: budgets.max_observations_per_artifact,
109 }
110 .into());
111 }
112 let observed = u64::try_from(stored.payload.len()).unwrap_or(u64::MAX);
113 if observed > budgets.max_serialized_output_bytes_per_artifact {
114 return Err(ExtractionLimitExceeded {
115 artifact: stored.source.path.display.clone(),
116 extractor: stored.source.extractor.clone(),
117 resource: ExtractionResource::SerializedOutputBytes,
118 observed,
119 maximum: budgets.max_serialized_output_bytes_per_artifact,
120 }
121 .into());
122 }
123 let outputs: Vec<T> = serde_json::from_slice(&stored.payload)
124 .map_err(|error| BatchPlanError::InvalidPayload(error.to_string()))?;
125 if usize::try_from(stored.output_count).ok() != Some(outputs.len()) {
126 return Err(BatchPlanError::OutputCountMismatch {
127 stored: stored.output_count,
128 decoded: outputs.len(),
129 });
130 }
131 Ok(ExtractorBatch::new(stored.source.clone(), outputs))
132}
133
134#[cfg(test)]
135mod tests {
136 use code_system_graph_model::{
137 ArtifactFingerprint, CheckoutId, NativePath, NativePathEncoding, RepoId
138 };
139
140 use super::{
141 BatchPlanError, ExtractorBatch, load_extractor_batch, load_extractor_batch_with_budgets, store_extractor_batch
142 };
143 use crate::{ExtractionBudgets, ExtractionLimitExceeded, ExtractionTracker};
144
145 fn tracker() -> ExtractionTracker {
146 ExtractionTracker::new("src/routes.rs", "test", &ExtractionBudgets::default())
147 }
148
149 fn path(value: &str) -> NativePath {
150 NativePath {
151 encoding: NativePathEncoding::Utf8,
152 bytes: value.as_bytes().to_vec(),
153 display: value.to_owned(),
154 }
155 }
156
157 fn batch(source: &str, hash: &str, outputs: &[&str]) -> ExtractorBatch<String> {
158 ExtractorBatch::new(
159 ArtifactFingerprint {
160 repo_id: RepoId::new("repo:api"),
161 checkout_id: CheckoutId::new("checkout:api"),
162 path: path(source),
163 extractor: "code-system-graph.http.openapi".to_owned(),
164 content_hash: hash.to_owned(),
165 size_bytes: 1,
166 },
167 outputs.iter().map(|output| (*output).to_owned()).collect(),
168 )
169 }
170
171 #[test]
172 fn stored_batch_should_round_trip_without_source_text() {
173 let original = batch("src/routes.rs", "hash", &["GET:/orders", "POST:/orders"]);
174 let result = store_extractor_batch(&original, &mut tracker(), false)
175 .and_then(|stored| load_extractor_batch::<String>(&stored));
176
177 assert_eq!(result, Ok(original));
178 }
179
180 #[test]
181 fn stored_batch_should_reject_inconsistent_output_count() {
182 let original = batch("src/routes.rs", "hash", &["GET:/orders"]);
183 let result =
184 store_extractor_batch(&original, &mut tracker(), false).and_then(|mut stored| {
185 stored.output_count = 2;
186 load_extractor_batch::<String>(&stored)
187 });
188
189 assert!(matches!(
190 result,
191 Err(BatchPlanError::OutputCountMismatch {
192 stored: 2,
193 decoded: 1
194 })
195 ));
196 }
197
198 #[test]
199 fn stored_batch_should_check_payload_limit_before_decoding() {
200 let original = batch("src/routes.rs", "hash", &["GET:/orders"]);
201 let stored = store_extractor_batch(&original, &mut tracker(), false).expect("stored batch");
202 let exact = u64::try_from(stored.payload.len()).expect("payload length");
203 let exact_budgets = ExtractionBudgets {
204 max_serialized_output_bytes_per_artifact: exact,
205 ..ExtractionBudgets::default()
206 };
207 let below_budgets = ExtractionBudgets {
208 max_serialized_output_bytes_per_artifact: exact - 1,
209 ..ExtractionBudgets::default()
210 };
211
212 assert_eq!(
213 load_extractor_batch_with_budgets::<String>(&stored, &exact_budgets),
214 Ok(original)
215 );
216 assert!(matches!(
217 load_extractor_batch_with_budgets::<String>(&stored, &below_budgets),
218 Err(BatchPlanError::ExtractionLimit(ExtractionLimitExceeded {
219 resource: crate::ExtractionResource::SerializedOutputBytes,
220 observed,
221 maximum,
222 ..
223 })) if observed == exact && maximum == exact - 1
224 ));
225 }
226
227 #[test]
228 fn stored_batch_should_check_observation_limit_before_decoding() {
229 let original = batch("src/routes.rs", "hash", &["GET:/orders", "POST:/orders"]);
230 let stored = store_extractor_batch(&original, &mut tracker(), false).expect("stored batch");
231 let budgets = ExtractionBudgets {
232 max_observations_per_artifact: 1,
233 ..ExtractionBudgets::default()
234 };
235
236 assert!(matches!(
237 load_extractor_batch_with_budgets::<String>(&stored, &budgets),
238 Err(BatchPlanError::ExtractionLimit(ExtractionLimitExceeded {
239 resource: crate::ExtractionResource::Observations,
240 observed: 2,
241 maximum: 1,
242 ..
243 }))
244 ));
245 }
246}