1use std::io::Cursor;
2
3use shardline_index::{FileRecord, parse_xet_hash_hex};
4use shardline_protocol::{ByteRange, ShardlineHash};
5
6use crate::{
7 BazelCacheKind, InvalidReconstructionResponseError, InvalidSerializedShardError, ServerError,
8 app::{parse_oci_path, parse_upload_content_range},
9 bazel_cache_object_key,
10 config::ShardMetadataLimits,
11 lfs_object_key,
12 lifecycle_repair::{
13 QuarantineRepairAction, RetentionHoldRepairAction, WebhookDeliveryRepairAction,
14 classify_quarantine_repair_action, classify_retention_hold_repair_action,
15 classify_webhook_delivery_repair_action,
16 },
17 oci_adapter::{oci_blob_key, oci_manifest_key, parse_reference},
18 protocol_support::{parse_sha256_digest, validate_oci_repository_name, validate_oci_tag},
19 server_frontend::ServerFrontend,
20 xet_adapter::{
21 build_reconstruction_response, build_xorb_transfer_url, normalize_serialized_xorb,
22 reconstruction_v2_from_v1, retained_shard_chunk_hashes, validate_serialized_xorb,
23 },
24};
25
26#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct FuzzLfsFrontendSummary {
29 pub oid_accepts: bool,
31 pub key_is_stable: bool,
33}
34
35#[derive(Debug, Clone, PartialEq, Eq)]
37pub struct FuzzBazelHttpFrontendSummary {
38 pub ac_accepts: bool,
40 pub cas_accepts: bool,
42}
43
44#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct FuzzOciFrontendSummary {
47 pub repository_accepts: bool,
49 pub reference_accepts: bool,
51 pub digest_accepts: bool,
53 pub session_accepts: bool,
55 pub content_range_accepts: bool,
57 pub path_accepts: bool,
59 pub blob_accepts: bool,
61 pub manifest_accepts: bool,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq)]
67pub struct FuzzValidatedXorbSummary {
68 pub normalized_len: u64,
70 pub total_len: u64,
72 pub packed_content_len: u64,
74 pub unpacked_len: u64,
76 pub chunk_count: usize,
78}
79
80pub fn fuzz_normalize_and_validate_xorb(
88 expected_hash: ShardlineHash,
89 bytes: &[u8],
90) -> Result<FuzzValidatedXorbSummary, ServerError> {
91 let normalized = normalize_serialized_xorb(expected_hash, bytes)?;
92 let normalized_len = u64::try_from(normalized.len())?;
93 let mut cursor = Cursor::new(normalized.as_slice());
94 let validated =
95 validate_serialized_xorb(&mut cursor, expected_hash).map_err(ServerError::from)?;
96
97 Ok(FuzzValidatedXorbSummary {
98 normalized_len,
99 total_len: validated.total_length(),
100 packed_content_len: validated.packed_content_length(),
101 unpacked_len: validated.unpacked_length(),
102 chunk_count: validated.chunks().len(),
103 })
104}
105
106#[derive(Debug, Clone, PartialEq, Eq)]
108pub struct FuzzRetainedShardSummary {
109 pub dedupe_chunk_hashes: Vec<String>,
111}
112
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
115pub struct FuzzLifecycleRepairSummary {
116 pub quarantine_keep: u64,
118 pub quarantine_delete_missing: u64,
120 pub quarantine_delete_reachable: u64,
122 pub quarantine_delete_held: u64,
124 pub retention_keep: u64,
126 pub retention_delete_expired: u64,
128 pub retention_delete_missing: u64,
130 pub webhook_keep: u64,
132 pub webhook_delete_stale: u64,
134 pub webhook_delete_future: u64,
136}
137
138#[derive(Debug, Clone, PartialEq, Eq)]
140pub struct FuzzReconstructionResponseSummary {
141 pub terms: usize,
143 pub fetch_xorbs: usize,
145 pub fetch_ranges: usize,
147 pub v2_xorbs: usize,
149 pub v2_fetches: usize,
151 pub v2_ranges: usize,
153 pub offset_into_first_range: u64,
155 pub total_unpacked_length: u64,
157}
158
159#[derive(Debug, Clone, PartialEq, Eq)]
161pub struct FuzzProtocolFrontendSummary {
162 pub frontend_accepts: bool,
164 pub digest_accepts: bool,
166 pub lfs_accepts: bool,
168 pub bazel_accepts: bool,
170 pub oci_repository_accepts: bool,
172 pub oci_reference_accepts: bool,
174 pub oci_blob_accepts: bool,
176 pub oci_manifest_accepts: bool,
178}
179
180pub fn fuzz_reconstruction_response_summary(
187 public_base_url: &str,
188 record: &FileRecord,
189 requested_range: Option<ByteRange>,
190) -> Result<FuzzReconstructionResponseSummary, ServerError> {
191 let response = build_reconstruction_response(public_base_url, record, requested_range)?;
192 ensure_reconstruction_response_invariant(
193 response.terms.len() <= record.chunks.len(),
194 InvalidReconstructionResponseError::TermCountExceededRecordChunkCount,
195 )?;
196
197 let mut total_unpacked_length = 0_u64;
198 for term in &response.terms {
199 parse_xet_hash_hex(&term.hash)?;
200 ensure_reconstruction_response_invariant(
201 term.unpacked_length > 0,
202 InvalidReconstructionResponseError::TermHadZeroUnpackedLength,
203 )?;
204 ensure_reconstruction_response_invariant(
205 term.range.start < term.range.end,
206 InvalidReconstructionResponseError::TermHadEmptyChunkRange,
207 )?;
208 ensure_reconstruction_response_invariant(
209 response.fetch_info.contains_key(&term.hash),
210 InvalidReconstructionResponseError::TermMissingFetchInfo,
211 )?;
212 let next_total = total_unpacked_length
213 .checked_add(term.unpacked_length)
214 .ok_or(ServerError::Overflow)?;
215 total_unpacked_length = next_total;
216 }
217
218 let mut fetch_ranges = 0_usize;
219 for (hash, fetch_entries) in &response.fetch_info {
220 parse_xet_hash_hex(hash)?;
221 ensure_reconstruction_response_invariant(
222 !fetch_entries.is_empty(),
223 InvalidReconstructionResponseError::EmptyFetchList,
224 )?;
225 for fetch_entry in fetch_entries {
226 ensure_reconstruction_response_invariant(
227 fetch_entry.url == build_xorb_transfer_url(public_base_url, hash),
228 InvalidReconstructionResponseError::FetchUrlHashMismatch,
229 )?;
230 ensure_reconstruction_response_invariant(
231 fetch_entry.range.start < fetch_entry.range.end,
232 InvalidReconstructionResponseError::FetchEntryEmptyChunkRange,
233 )?;
234 ensure_reconstruction_response_invariant(
235 fetch_entry.url_range.start <= fetch_entry.url_range.end,
236 InvalidReconstructionResponseError::FetchEntryInvertedByteRange,
237 )?;
238 ensure_reconstruction_response_invariant(
239 response.terms.iter().any(|term| {
240 term.hash == *hash
241 && term.range == fetch_entry.range
242 && term.unpacked_length > 0
243 }),
244 InvalidReconstructionResponseError::FetchEntryMissingTerm,
245 )?;
246 fetch_ranges = fetch_ranges.checked_add(1).ok_or(ServerError::Overflow)?;
247 }
248 }
249
250 let v2 = reconstruction_v2_from_v1(response.clone());
251 ensure_reconstruction_response_invariant(
252 v2.offset_into_first_range == response.offset_into_first_range,
253 InvalidReconstructionResponseError::V2ChangedOffsetIntoFirstRange,
254 )?;
255 ensure_reconstruction_response_invariant(
256 v2.terms == response.terms,
257 InvalidReconstructionResponseError::V2ChangedTerms,
258 )?;
259 ensure_reconstruction_response_invariant(
260 v2.xorbs.len() == response.fetch_info.len(),
261 InvalidReconstructionResponseError::V2ChangedXorbFetchInfoCardinality,
262 )?;
263
264 let mut v2_fetches = 0_usize;
265 let mut v2_ranges = 0_usize;
266 for (hash, entries) in &v2.xorbs {
267 ensure_reconstruction_response_invariant(
268 response.fetch_info.contains_key(hash),
269 InvalidReconstructionResponseError::V2FetchHashAbsentFromV1,
270 )?;
271 ensure_reconstruction_response_invariant(
272 !entries.is_empty(),
273 InvalidReconstructionResponseError::V2EmptyFetchList,
274 )?;
275 v2_fetches = v2_fetches
276 .checked_add(entries.len())
277 .ok_or(ServerError::Overflow)?;
278 for entry in entries {
279 ensure_reconstruction_response_invariant(
280 entry.url == build_xorb_transfer_url(public_base_url, hash),
281 InvalidReconstructionResponseError::FetchUrlHashMismatch,
282 )?;
283 ensure_reconstruction_response_invariant(
284 !entry.ranges.is_empty(),
285 InvalidReconstructionResponseError::V2FetchEntryWithoutRanges,
286 )?;
287 v2_ranges = v2_ranges
288 .checked_add(entry.ranges.len())
289 .ok_or(ServerError::Overflow)?;
290 for range in &entry.ranges {
291 ensure_reconstruction_response_invariant(
292 range.chunks.start < range.chunks.end,
293 InvalidReconstructionResponseError::V2EmptyChunkRange,
294 )?;
295 ensure_reconstruction_response_invariant(
296 range.bytes.start <= range.bytes.end,
297 InvalidReconstructionResponseError::V2InvertedByteRange,
298 )?;
299 }
300 }
301 }
302 ensure_reconstruction_response_invariant(
303 v2_fetches == fetch_ranges,
304 InvalidReconstructionResponseError::V2FetchCountDisagreedWithV1,
305 )?;
306 ensure_reconstruction_response_invariant(
307 v2_ranges == fetch_ranges,
308 InvalidReconstructionResponseError::V2RangeCountDisagreedWithV1,
309 )?;
310
311 Ok(FuzzReconstructionResponseSummary {
312 terms: response.terms.len(),
313 fetch_xorbs: response.fetch_info.len(),
314 fetch_ranges,
315 v2_xorbs: v2.xorbs.len(),
316 v2_fetches,
317 v2_ranges,
318 offset_into_first_range: response.offset_into_first_range,
319 total_unpacked_length,
320 })
321}
322
323pub fn fuzz_protocol_frontend_summary(
330 frontend: &str,
331 oid: &str,
332 digest: &str,
333 repository: &str,
334 reference: &str,
335) -> Result<FuzzProtocolFrontendSummary, ServerError> {
336 let frontend_accepts = ServerFrontend::parse(frontend).is_ok();
337 let digest_accepts = parse_sha256_digest(digest).is_ok();
338
339 let lfs_accepts = match lfs_object_key(oid, None) {
340 Ok(key) => {
341 let repeated = lfs_object_key(oid, None)?;
342 key.as_str() == repeated.as_str()
343 }
344 Err(_) => false,
345 };
346
347 let bazel_accepts = match bazel_cache_object_key(BazelCacheKind::Cas, oid, None) {
348 Ok(key) => {
349 let repeated = bazel_cache_object_key(BazelCacheKind::Cas, oid, None)?;
350 key.as_str() == repeated.as_str()
351 }
352 Err(_) => false,
353 };
354
355 let oci_repository_accepts = validate_oci_repository_name(repository).is_ok();
356 let oci_reference_accepts =
357 parse_reference(reference).is_ok() || validate_oci_tag(reference).is_ok();
358
359 let digest_hex = parse_sha256_digest(digest).ok();
360 let oci_blob_accepts = if let Some(digest_hex) = digest_hex.as_deref() {
361 match oci_blob_key(repository, digest_hex, None) {
362 Ok(key) => {
363 let repeated = oci_blob_key(repository, digest_hex, None)?;
364 key.as_str() == repeated.as_str()
365 }
366 Err(_) => false,
367 }
368 } else {
369 false
370 };
371 let oci_manifest_accepts = if let Some(digest_hex) = digest_hex.as_deref() {
372 match oci_manifest_key(repository, digest_hex, None) {
373 Ok(key) => {
374 let repeated = oci_manifest_key(repository, digest_hex, None)?;
375 key.as_str() == repeated.as_str()
376 }
377 Err(_) => false,
378 }
379 } else {
380 false
381 };
382
383 Ok(FuzzProtocolFrontendSummary {
384 frontend_accepts,
385 digest_accepts,
386 lfs_accepts,
387 bazel_accepts,
388 oci_repository_accepts,
389 oci_reference_accepts,
390 oci_blob_accepts,
391 oci_manifest_accepts,
392 })
393}
394
395pub fn fuzz_lfs_frontend_summary(oid: &str) -> Result<FuzzLfsFrontendSummary, ServerError> {
402 let (oid_accepts, key_is_stable) = match lfs_object_key(oid, None) {
403 Ok(key) => {
404 let repeated = lfs_object_key(oid, None)?;
405 (true, key.as_str() == repeated.as_str())
406 }
407 Err(_) => (false, false),
408 };
409
410 Ok(FuzzLfsFrontendSummary {
411 oid_accepts,
412 key_is_stable,
413 })
414}
415
416pub fn fuzz_bazel_http_frontend_summary(
423 hash_hex: &str,
424) -> Result<FuzzBazelHttpFrontendSummary, ServerError> {
425 let ac_accepts = match bazel_cache_object_key(BazelCacheKind::Ac, hash_hex, None) {
426 Ok(key) => {
427 let repeated = bazel_cache_object_key(BazelCacheKind::Ac, hash_hex, None)?;
428 key.as_str() == repeated.as_str()
429 }
430 Err(_) => false,
431 };
432 let cas_accepts = match bazel_cache_object_key(BazelCacheKind::Cas, hash_hex, None) {
433 Ok(key) => {
434 let repeated = bazel_cache_object_key(BazelCacheKind::Cas, hash_hex, None)?;
435 key.as_str() == repeated.as_str()
436 }
437 Err(_) => false,
438 };
439
440 Ok(FuzzBazelHttpFrontendSummary {
441 ac_accepts,
442 cas_accepts,
443 })
444}
445
446pub fn fuzz_oci_frontend_summary(
453 repository: &str,
454 reference: &str,
455 digest: &str,
456 session_id: &str,
457 content_range: &str,
458 path: &str,
459) -> Result<FuzzOciFrontendSummary, ServerError> {
460 let repository_accepts = validate_oci_repository_name(repository).is_ok();
461 let reference_accepts = parse_reference(reference).is_ok();
462 let digest_accepts = parse_sha256_digest(digest).is_ok();
463 let session_accepts = crate::protocol_support::validate_upload_session_id(session_id).is_ok();
464 let content_range_accepts = parse_upload_content_range(content_range).is_ok();
465 let path_accepts = parse_oci_path(path).is_ok();
466 let digest_hex = parse_sha256_digest(digest).ok();
467 let blob_accepts = digest_hex
468 .as_deref()
469 .is_some_and(|digest_hex| oci_blob_key(repository, digest_hex, None).is_ok());
470 let manifest_accepts = digest_hex
471 .as_deref()
472 .is_some_and(|digest_hex| oci_manifest_key(repository, digest_hex, None).is_ok());
473
474 Ok(FuzzOciFrontendSummary {
475 repository_accepts,
476 reference_accepts,
477 digest_accepts,
478 session_accepts,
479 content_range_accepts,
480 path_accepts,
481 blob_accepts,
482 manifest_accepts,
483 })
484}
485
486fn ensure_reconstruction_response_invariant(
487 condition: bool,
488 error: InvalidReconstructionResponseError,
489) -> Result<(), ServerError> {
490 if condition { Ok(()) } else { Err(error.into()) }
491}
492
493pub fn fuzz_lifecycle_repair_summary(
499 now_unix_seconds: u64,
500 webhook_retention_seconds: u64,
501 quarantine_states: &[(bool, bool, bool)],
502 retention_states: &[(Option<u64>, u64, bool)],
503 webhook_processed_at_unix_seconds: &[u64],
504) -> Result<FuzzLifecycleRepairSummary, ServerError> {
505 let max_processed_at_unix_seconds = now_unix_seconds
506 .checked_add(300)
507 .ok_or(ServerError::Overflow)?;
508 let stale_cutoff_unix_seconds = now_unix_seconds.saturating_sub(webhook_retention_seconds);
509
510 let mut summary = FuzzLifecycleRepairSummary {
511 quarantine_keep: 0,
512 quarantine_delete_missing: 0,
513 quarantine_delete_reachable: 0,
514 quarantine_delete_held: 0,
515 retention_keep: 0,
516 retention_delete_expired: 0,
517 retention_delete_missing: 0,
518 webhook_keep: 0,
519 webhook_delete_stale: 0,
520 webhook_delete_future: 0,
521 };
522
523 for &(object_exists, is_reachable, is_held) in quarantine_states {
524 match classify_quarantine_repair_action(object_exists, is_reachable, is_held) {
525 QuarantineRepairAction::Keep => {
526 summary.quarantine_keep = increment_counter(summary.quarantine_keep)?;
527 }
528 QuarantineRepairAction::DeleteMissing => {
529 summary.quarantine_delete_missing =
530 increment_counter(summary.quarantine_delete_missing)?;
531 }
532 QuarantineRepairAction::DeleteReachable => {
533 summary.quarantine_delete_reachable =
534 increment_counter(summary.quarantine_delete_reachable)?;
535 }
536 QuarantineRepairAction::DeleteHeld => {
537 summary.quarantine_delete_held = increment_counter(summary.quarantine_delete_held)?;
538 }
539 }
540 }
541
542 for &(release_after_unix_seconds, held_at_unix_seconds, object_exists) in retention_states {
543 match classify_retention_hold_repair_action(
544 release_after_unix_seconds,
545 held_at_unix_seconds,
546 object_exists,
547 now_unix_seconds,
548 ) {
549 RetentionHoldRepairAction::Keep => {
550 summary.retention_keep = increment_counter(summary.retention_keep)?;
551 }
552 RetentionHoldRepairAction::DeleteExpired => {
553 summary.retention_delete_expired =
554 increment_counter(summary.retention_delete_expired)?;
555 }
556 RetentionHoldRepairAction::DeleteMissing => {
557 summary.retention_delete_missing =
558 increment_counter(summary.retention_delete_missing)?;
559 }
560 }
561 }
562
563 for &processed_at_unix_seconds in webhook_processed_at_unix_seconds {
564 match classify_webhook_delivery_repair_action(
565 processed_at_unix_seconds,
566 stale_cutoff_unix_seconds,
567 max_processed_at_unix_seconds,
568 ) {
569 WebhookDeliveryRepairAction::Keep => {
570 summary.webhook_keep = increment_counter(summary.webhook_keep)?;
571 }
572 WebhookDeliveryRepairAction::DeleteStale => {
573 summary.webhook_delete_stale = increment_counter(summary.webhook_delete_stale)?;
574 }
575 WebhookDeliveryRepairAction::DeleteFuture => {
576 summary.webhook_delete_future = increment_counter(summary.webhook_delete_future)?;
577 }
578 }
579 }
580
581 Ok(summary)
582}
583
584pub fn fuzz_retained_shard_chunk_hashes(
593 shard_bytes: &[u8],
594 limits: ShardMetadataLimits,
595) -> Result<FuzzRetainedShardSummary, ServerError> {
596 let dedupe_chunk_hashes = retained_shard_chunk_hashes(shard_bytes, limits)?;
597 for window in dedupe_chunk_hashes.windows(2) {
598 let [left, right] = window else {
599 continue;
600 };
601 if left >= right {
602 return Err(
603 InvalidSerializedShardError::RetainedShardChunkHashesNotStrictlyOrdered.into(),
604 );
605 }
606 }
607 for hash in &dedupe_chunk_hashes {
608 parse_xet_hash_hex(hash).map_err(ServerError::from)?;
609 }
610
611 Ok(FuzzRetainedShardSummary {
612 dedupe_chunk_hashes,
613 })
614}
615
616fn increment_counter(value: u64) -> Result<u64, ServerError> {
617 value.checked_add(1).ok_or(ServerError::Overflow)
618}