1mod fidelity;
16mod integrity;
17mod metadata;
18mod structure;
19pub mod types;
20
21use std::path::Path;
22
23use self::integrity::validate_integrity;
24use self::metadata::validate_metadata;
25use self::structure::validate_structure;
26pub use self::types::*;
27
28pub fn validate_message(buf: &[u8], options: &ValidateOptions) -> ValidationReport {
34 let mut issues = Vec::new();
35 let mut object_count = 0;
36 let mut hash_verified = false;
37
38 let effective_max = if options.checksum_only {
40 options.max_level.max(ValidationLevel::Integrity)
41 } else {
42 options.max_level
43 };
44 let report_structure = !options.checksum_only;
45 let check_canonical = options.check_canonical;
46 let run_metadata =
48 (effective_max >= ValidationLevel::Metadata && !options.checksum_only) || check_canonical;
49 let run_integrity = effective_max >= ValidationLevel::Integrity;
50 let run_fidelity = effective_max >= ValidationLevel::Fidelity && !options.checksum_only;
51
52 let mut structure_issues = Vec::new();
57 let walk = validate_structure(buf, &mut structure_issues);
58 if report_structure {
59 issues.append(&mut structure_issues);
61 } else {
62 issues.extend(
66 structure_issues
67 .into_iter()
68 .filter(|i| i.severity == IssueSeverity::Error),
69 );
70 }
71
72 if let Some(ref walk) = walk {
73 object_count = walk.data_objects.len();
74
75 let needs_objects = run_metadata || run_integrity || run_fidelity || check_canonical;
78 let mut objects: Vec<ObjectContext<'_>> = if needs_objects {
79 walk.data_objects
80 .iter()
81 .map(|(cbor_bytes, payload, frame_offset)| ObjectContext {
82 descriptor: None,
83 descriptor_failed: false,
84 cbor_bytes,
85 payload,
86 frame_offset: *frame_offset,
87 decode_state: DecodeState::NotDecoded,
88 })
89 .collect()
90 } else {
91 Vec::new()
92 };
93
94 if run_metadata {
96 validate_metadata(walk, &mut objects, &mut issues, check_canonical);
97 }
98
99 if run_integrity {
101 hash_verified = validate_integrity(
102 walk,
103 &mut objects,
104 &mut issues,
105 options.checksum_only,
106 run_fidelity,
107 );
108 }
109
110 if run_fidelity {
112 fidelity::validate_fidelity(&mut objects, &mut issues);
113 }
114 }
115
116 if issues.iter().any(|i| i.severity == IssueSeverity::Error) {
119 hash_verified = false;
120 }
121
122 ValidationReport {
123 issues,
124 object_count,
125 hash_verified,
126 }
127}
128
129pub fn validate_file(
134 path: &Path,
135 options: &ValidateOptions,
136) -> std::io::Result<FileValidationReport> {
137 use std::io::{Read, Seek, SeekFrom};
138
139 let file_len = usize::try_from(std::fs::metadata(path)?.len()).map_err(|_| {
140 std::io::Error::new(
141 std::io::ErrorKind::InvalidData,
142 "file size does not fit into usize",
143 )
144 })?;
145 let mut file = std::fs::File::open(path)?;
146
147 let offsets = crate::framing::scan_file(&mut file)
148 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
149
150 let mut file_issues = Vec::new();
151 let mut messages = Vec::new();
152 let mut expected_pos: usize = 0;
153
154 for (offset, length) in &offsets {
155 if *offset > expected_pos {
156 file_issues.push(FileIssue {
157 byte_offset: expected_pos,
158 length: offset - expected_pos,
159 description: format!(
160 "{} unrecognized bytes at offset {}",
161 offset - expected_pos,
162 expected_pos
163 ),
164 });
165 }
166
167 file.seek(SeekFrom::Start(*offset as u64))?;
168 let mut msg_buf = vec![0u8; *length];
169 file.read_exact(&mut msg_buf)?;
170
171 let report = validate_message(&msg_buf, options);
172 messages.push(report);
173
174 expected_pos = offset + length;
175 }
176
177 if expected_pos < file_len {
178 let trailing_len = file_len - expected_pos;
179 let desc = if messages.is_empty() {
180 format!("{trailing_len} bytes with no valid messages")
181 } else {
182 format!("{trailing_len} trailing bytes after last message at offset {expected_pos}")
183 };
184 file_issues.push(FileIssue {
185 byte_offset: expected_pos,
186 length: trailing_len,
187 description: desc,
188 });
189 }
190
191 Ok(FileValidationReport {
192 file_issues,
193 messages,
194 })
195}
196
197pub fn validate_buffer(buf: &[u8], options: &ValidateOptions) -> FileValidationReport {
201 let offsets = crate::framing::scan(buf);
202
203 let mut file_issues = Vec::new();
204 let mut messages = Vec::new();
205 let mut expected_pos: usize = 0;
206
207 for (offset, length) in &offsets {
208 if *offset > expected_pos {
209 file_issues.push(FileIssue {
210 byte_offset: expected_pos,
211 length: offset - expected_pos,
212 description: format!(
213 "{} unrecognized bytes at offset {}",
214 offset - expected_pos,
215 expected_pos
216 ),
217 });
218 }
219
220 let msg_slice = &buf[*offset..*offset + *length];
221 let report = validate_message(msg_slice, options);
222 messages.push(report);
223
224 expected_pos = offset + length;
225 }
226
227 if expected_pos < buf.len() {
228 let trailing_len = buf.len() - expected_pos;
229 let desc = if messages.is_empty() {
230 format!("{trailing_len} bytes with no valid messages")
231 } else {
232 format!("{trailing_len} trailing bytes after last message at offset {expected_pos}")
233 };
234 file_issues.push(FileIssue {
235 byte_offset: expected_pos,
236 length: trailing_len,
237 description: desc,
238 });
239 }
240
241 FileValidationReport {
242 file_issues,
243 messages,
244 }
245}
246
247#[cfg(test)]
248mod tests {
249 use super::*;
250 use crate::Dtype;
251 use crate::encode::{EncodeOptions, encode};
252 use crate::types::{DataObjectDescriptor, GlobalMetadata};
253 use crate::wire::{FRAME_HEADER_SIZE, POSTAMBLE_SIZE, PREAMBLE_SIZE};
254 use std::collections::BTreeMap;
255 use tensogram_encodings::ByteOrder;
256
257 fn make_test_message() -> Vec<u8> {
258 let meta = GlobalMetadata::default();
259 let desc = DataObjectDescriptor {
260 obj_type: "ndarray".to_string(),
261 ndim: 1,
262 shape: vec![4],
263 strides: vec![8],
264 dtype: Dtype::Float64,
265 byte_order: ByteOrder::Big,
266 encoding: "none".to_string(),
267 filter: "none".to_string(),
268 compression: "none".to_string(),
269 params: BTreeMap::new(),
270 hash: None,
271 };
272 let data: Vec<u8> = vec![0u8; 32];
273 encode(
274 &meta,
275 &[(&desc, data.as_slice())],
276 &EncodeOptions::default(),
277 )
278 .unwrap()
279 }
280
281 fn make_multi_object_message() -> Vec<u8> {
282 let meta = GlobalMetadata::default();
283 let desc = DataObjectDescriptor {
284 obj_type: "ndarray".to_string(),
285 ndim: 1,
286 shape: vec![2],
287 strides: vec![8],
288 dtype: Dtype::Float64,
289 byte_order: ByteOrder::Big,
290 encoding: "none".to_string(),
291 filter: "none".to_string(),
292 compression: "none".to_string(),
293 params: BTreeMap::new(),
294 hash: None,
295 };
296 let data: Vec<u8> = vec![0u8; 16];
297 encode(
298 &meta,
299 &[(&desc, data.as_slice()), (&desc, data.as_slice())],
300 &EncodeOptions::default(),
301 )
302 .unwrap()
303 }
304
305 #[test]
308 fn valid_message_passes_all_levels() {
309 let msg = make_test_message();
310 let report = validate_message(&msg, &ValidateOptions::default());
311 assert!(report.is_ok(), "issues: {:?}", report.issues);
312 assert_eq!(report.object_count, 1);
313 assert!(report.hash_verified);
314 }
315
316 #[test]
317 fn valid_multi_object_passes() {
318 let msg = make_multi_object_message();
319 let report = validate_message(&msg, &ValidateOptions::default());
320 assert!(report.is_ok(), "issues: {:?}", report.issues);
321 assert_eq!(report.object_count, 2);
322 }
323
324 #[test]
325 fn empty_buffer_fails() {
326 let report = validate_message(&[], &ValidateOptions::default());
327 assert!(!report.is_ok());
328 assert_eq!(report.issues[0].code, IssueCode::BufferTooShort);
329 }
330
331 #[test]
332 fn wrong_magic_fails() {
333 let mut msg = make_test_message();
334 msg[0..8].copy_from_slice(b"WRONGMAG");
335 let report = validate_message(&msg, &ValidateOptions::default());
336 assert!(!report.is_ok());
337 assert_eq!(report.issues[0].code, IssueCode::InvalidMagic);
338 }
339
340 #[test]
341 fn truncated_message_fails() {
342 let msg = make_test_message();
343 let truncated = &msg[..msg.len() / 2];
344 let report = validate_message(truncated, &ValidateOptions::default());
345 assert!(!report.is_ok());
346 }
347
348 #[test]
349 fn bad_total_length_fails() {
350 let mut msg = make_test_message();
351 let bad_len: u64 = (msg.len() * 10) as u64;
352 msg[16..24].copy_from_slice(&bad_len.to_be_bytes());
353 let report = validate_message(&msg, &ValidateOptions::default());
354 assert!(!report.is_ok());
355 assert_eq!(report.issues[0].code, IssueCode::TotalLengthExceedsBuffer);
356 }
357
358 #[test]
359 fn corrupted_postamble_fails() {
360 let mut msg = make_test_message();
361 let end = msg.len();
362 msg[end - 8..end].copy_from_slice(b"BADMAGIC");
363 let report = validate_message(&msg, &ValidateOptions::default());
364 assert!(!report.is_ok());
365 }
366
367 #[test]
368 fn tiny_total_length_does_not_panic() {
369 let mut buf = vec![0u8; 40];
370 buf[0..8].copy_from_slice(b"TENSOGRM");
371 buf[8..10].copy_from_slice(&2u16.to_be_bytes());
372 buf[16..24].copy_from_slice(&10u64.to_be_bytes());
373 let report = validate_message(&buf, &ValidateOptions::default());
374 assert!(!report.is_ok());
375 assert_eq!(report.issues[0].code, IssueCode::TotalLengthTooSmall);
376 }
377
378 #[test]
381 fn corrupted_metadata_cbor_fails_level2() {
382 let mut msg = make_test_message();
383 let cbor_start = PREAMBLE_SIZE + FRAME_HEADER_SIZE;
384 if cbor_start + 4 < msg.len() {
385 msg[cbor_start] = 0xFF;
386 msg[cbor_start + 1] = 0xFF;
387 msg[cbor_start + 2] = 0xFF;
388 msg[cbor_start + 3] = 0xFF;
389 }
390 let report = validate_message(&msg, &ValidateOptions::default());
391 let has_meta_error = report
392 .issues
393 .iter()
394 .any(|i| i.level == ValidationLevel::Metadata);
395 assert!(
396 has_meta_error,
397 "expected metadata error, got: {:?}",
398 report.issues
399 );
400 }
401
402 #[test]
405 fn corrupted_byte_detected() {
406 let mut msg = make_test_message();
407 let target = PREAMBLE_SIZE + FRAME_HEADER_SIZE + 20;
410 if target < msg.len() {
411 msg[target] ^= 0xFF;
412 }
413 let report = validate_message(&msg, &ValidateOptions::default());
414 assert!(
415 !report.is_ok(),
416 "expected error after corruption, got: {:?}",
417 report.issues
418 );
419 }
420
421 #[test]
422 fn hash_mismatch_on_corrupted_payload() {
423 use crate::wire::POSTAMBLE_SIZE;
426 let msg = make_test_message();
427 let pa_start = msg.len() - POSTAMBLE_SIZE;
435 let target = pa_start * 7 / 10;
440 let mut corrupted = msg.clone();
441 corrupted[target] ^= 0xFF;
442 let report = validate_message(&corrupted, &ValidateOptions::default());
443 let has_hash_or_integrity = report.issues.iter().any(|i| {
444 matches!(
445 i.code,
446 IssueCode::HashMismatch | IssueCode::DecodePipelineFailed
447 )
448 });
449 assert!(
453 !report.is_ok(),
454 "corrupted payload should fail validation, got: {:?}",
455 report.issues
456 );
457 let _ = has_hash_or_integrity; }
461
462 #[test]
465 fn quick_mode_skips_metadata_and_integrity() {
466 let msg = make_test_message();
467 let opts = ValidateOptions {
468 max_level: ValidationLevel::Structure,
469 ..ValidateOptions::default()
470 };
471 let report = validate_message(&msg, &opts);
472 assert!(report.is_ok());
473 assert!(!report.hash_verified);
474 }
475
476 #[test]
477 fn checksum_mode_verifies_hash() {
478 let msg = make_test_message();
479 let opts = ValidateOptions {
480 checksum_only: true,
481 ..ValidateOptions::default()
482 };
483 let report = validate_message(&msg, &opts);
484 assert!(report.is_ok());
485 assert!(report.hash_verified);
486 }
487
488 #[test]
489 fn checksum_mode_on_broken_message_fails() {
490 let mut msg = make_test_message();
491 let end = msg.len();
492 msg[end - 8..end].copy_from_slice(b"BADMAGIC");
493 let opts = ValidateOptions {
494 checksum_only: true,
495 ..ValidateOptions::default()
496 };
497 let report = validate_message(&msg, &opts);
498 assert!(
499 !report.is_ok(),
500 "broken message should fail even in checksum mode"
501 );
502 }
503
504 #[test]
505 fn checksum_mode_catches_structural_errors() {
506 let mut msg = make_test_message();
507 let actual_len = msg.len() as u64;
508 let bad_len = actual_len + 100;
509 msg[16..24].copy_from_slice(&bad_len.to_be_bytes());
510 let opts = ValidateOptions {
511 checksum_only: true,
512 ..ValidateOptions::default()
513 };
514 let report = validate_message(&msg, &opts);
515 let has_error = report
516 .issues
517 .iter()
518 .any(|i| i.severity == IssueSeverity::Error);
519 assert!(
520 has_error,
521 "checksum mode should surface structural errors, got: {:?}",
522 report.issues
523 );
524 }
525
526 #[test]
527 fn canonical_mode_on_valid_message() {
528 let msg = make_test_message();
529 let opts = ValidateOptions {
530 check_canonical: true,
531 ..ValidateOptions::default()
532 };
533 let report = validate_message(&msg, &opts);
534 assert!(report.is_ok(), "issues: {:?}", report.issues);
535 }
536
537 #[test]
540 fn validate_buffer_single_message() {
541 let msg = make_test_message();
542 let report = validate_buffer(&msg, &ValidateOptions::default());
543 assert!(report.is_ok());
544 assert_eq!(report.messages.len(), 1);
545 assert!(report.file_issues.is_empty());
546 }
547
548 #[test]
549 fn validate_buffer_two_messages() {
550 let msg = make_test_message();
551 let mut buf = msg.clone();
552 buf.extend_from_slice(&msg);
553 let report = validate_buffer(&buf, &ValidateOptions::default());
554 assert!(report.is_ok());
555 assert_eq!(report.messages.len(), 2);
556 }
557
558 #[test]
559 fn validate_buffer_trailing_garbage() {
560 let mut buf = make_test_message();
561 buf.extend_from_slice(b"GARBAGE_TRAILING_DATA");
562 let report = validate_buffer(&buf, &ValidateOptions::default());
563 assert!(!report.file_issues.is_empty());
564 }
565
566 #[test]
567 fn validate_buffer_garbage_between_messages() {
568 let msg = make_test_message();
569 let mut buf = msg.clone();
570 buf.extend_from_slice(b"GARBAGE");
571 buf.extend_from_slice(&msg);
572 let report = validate_buffer(&buf, &ValidateOptions::default());
573 assert!(!report.file_issues.is_empty());
574 assert_eq!(report.messages.len(), 2);
575 }
576
577 #[test]
578 fn validate_buffer_truncated_second_message() {
579 let msg = make_test_message();
580 let mut buf = msg.clone();
581 buf.extend_from_slice(&msg[..msg.len() / 2]);
582 let report = validate_buffer(&buf, &ValidateOptions::default());
583 assert!(!report.messages.is_empty());
584 let has_issue =
585 !report.file_issues.is_empty() || report.messages.iter().any(|r| !r.is_ok());
586 assert!(
587 has_issue,
588 "truncated second message should produce an issue"
589 );
590 }
591
592 #[test]
595 fn streaming_message_validates() {
596 use crate::streaming::StreamingEncoder;
597
598 let meta = GlobalMetadata::default();
599 let desc = DataObjectDescriptor {
600 obj_type: "ndarray".to_string(),
601 ndim: 1,
602 shape: vec![4],
603 strides: vec![8],
604 dtype: Dtype::Float64,
605 byte_order: ByteOrder::Big,
606 encoding: "none".to_string(),
607 filter: "none".to_string(),
608 compression: "none".to_string(),
609 params: BTreeMap::new(),
610 hash: None,
611 };
612 let data = vec![0u8; 32];
613
614 let mut buf = Vec::new();
615 let mut enc = StreamingEncoder::new(&mut buf, &meta, &EncodeOptions::default()).unwrap();
616 enc.write_object(&desc, &data).unwrap();
617 enc.finish().unwrap();
618
619 let report = validate_message(&buf, &ValidateOptions::default());
620 assert!(
621 report.is_ok(),
622 "streaming message should validate: {:?}",
623 report.issues
624 );
625 assert_eq!(report.object_count, 1);
626 assert!(report.hash_verified);
627 }
628
629 #[test]
630 fn streaming_message_footer_metadata_validates() {
631 use crate::streaming::StreamingEncoder;
632
633 let meta = GlobalMetadata::default();
634 let desc = DataObjectDescriptor {
635 obj_type: "ndarray".to_string(),
636 ndim: 1,
637 shape: vec![2],
638 strides: vec![4],
639 dtype: Dtype::Float32,
640 byte_order: ByteOrder::Big,
641 encoding: "none".to_string(),
642 filter: "none".to_string(),
643 compression: "none".to_string(),
644 params: BTreeMap::new(),
645 hash: None,
646 };
647 let data = vec![0u8; 8];
648
649 let mut buf = Vec::new();
650 let mut enc = StreamingEncoder::new(&mut buf, &meta, &EncodeOptions::default()).unwrap();
651 enc.write_object(&desc, &data).unwrap();
652 enc.write_object(&desc, &data).unwrap();
653 enc.finish().unwrap();
654
655 let report = validate_message(&buf, &ValidateOptions::default());
656 assert!(report.is_ok(), "issues: {:?}", report.issues);
657 assert_eq!(report.object_count, 2);
658 }
659
660 #[test]
663 fn hash_verified_requires_all_objects() {
664 let msg = make_test_message();
665 let report = validate_message(&msg, &ValidateOptions::default());
666 assert!(report.hash_verified);
667 }
668
669 #[test]
670 fn hash_not_verified_without_hash() {
671 let meta = GlobalMetadata::default();
672 let desc = DataObjectDescriptor {
673 obj_type: "ndarray".to_string(),
674 ndim: 1,
675 shape: vec![4],
676 strides: vec![8],
677 dtype: Dtype::Float64,
678 byte_order: ByteOrder::Big,
679 encoding: "none".to_string(),
680 filter: "none".to_string(),
681 compression: "none".to_string(),
682 params: BTreeMap::new(),
683 hash: None,
684 };
685 let data = vec![0u8; 32];
686 let opts = EncodeOptions {
687 hash_algorithm: None,
688 ..EncodeOptions::default()
689 };
690 let msg = encode(&meta, &[(&desc, data.as_slice())], &opts).unwrap();
691 let report = validate_message(&msg, &ValidateOptions::default());
692 assert!(!report.hash_verified);
693 }
694
695 #[test]
698 fn issue_codes_are_stable_strings() {
699 let code = IssueCode::HashMismatch;
701 let json = serde_json::to_string(&code).unwrap();
702 assert_eq!(json, r#""hash_mismatch""#);
703 }
704
705 #[test]
706 fn report_serializes_to_json() {
707 let msg = make_test_message();
708 let report = validate_message(&msg, &ValidateOptions::default());
709 let json = serde_json::to_string(&report).unwrap();
710 assert!(json.contains("\"object_count\":1"));
711 assert!(json.contains("\"hash_verified\":true"));
712 }
713
714 #[test]
717 fn validate_buffer_garbage_only() {
718 let buf = b"this is not a tensogram file at all";
719 let report = validate_buffer(buf, &ValidateOptions::default());
720 assert!(report.messages.is_empty());
721 assert!(!report.file_issues.is_empty());
722 assert!(
723 report.file_issues[0]
724 .description
725 .contains("no valid messages"),
726 "got: {}",
727 report.file_issues[0].description,
728 );
729 }
730
731 #[test]
732 fn validate_buffer_empty() {
733 let report = validate_buffer(&[], &ValidateOptions::default());
734 assert!(report.messages.is_empty());
735 assert!(report.file_issues.is_empty());
736 assert!(report.is_ok());
737 }
738
739 #[test]
740 fn streaming_ffo_out_of_range_reported() {
741 use crate::streaming::StreamingEncoder;
743
744 let meta = GlobalMetadata::default();
745 let desc = DataObjectDescriptor {
746 obj_type: "ndarray".to_string(),
747 ndim: 1,
748 shape: vec![2],
749 strides: vec![8],
750 dtype: Dtype::Float64,
751 byte_order: ByteOrder::Big,
752 encoding: "none".to_string(),
753 filter: "none".to_string(),
754 compression: "none".to_string(),
755 params: BTreeMap::new(),
756 hash: None,
757 };
758 let data = vec![0u8; 16];
759
760 let mut buf = Vec::new();
761 let mut enc = StreamingEncoder::new(&mut buf, &meta, &EncodeOptions::default()).unwrap();
762 enc.write_object(&desc, &data).unwrap();
763 enc.finish().unwrap();
764
765 let pa_start = buf.len() - 16;
767 let bad_ffo: u64 = 0; buf[pa_start..pa_start + 8].copy_from_slice(&bad_ffo.to_be_bytes());
769
770 let report = validate_message(&buf, &ValidateOptions::default());
771 let has_ffo_error = report
772 .issues
773 .iter()
774 .any(|i| i.code == IssueCode::FooterOffsetOutOfRange);
775 assert!(
776 has_ffo_error,
777 "expected FooterOffsetOutOfRange, got: {:?}",
778 report.issues
779 );
780 }
781
782 #[test]
785 fn zero_object_message_validates() {
786 let meta = GlobalMetadata::default();
787 let opts = EncodeOptions {
788 hash_algorithm: None,
789 ..EncodeOptions::default()
790 };
791 let msg = encode(&meta, &[], &opts).unwrap();
792 let report = validate_message(&msg, &ValidateOptions::default());
793 assert!(report.is_ok(), "issues: {:?}", report.issues);
794 assert_eq!(report.object_count, 0);
795 assert!(!report.hash_verified); }
797
798 fn full_opts() -> ValidateOptions {
801 ValidateOptions {
802 max_level: ValidationLevel::Fidelity,
803 ..ValidateOptions::default()
804 }
805 }
806
807 fn make_float64_message(values: &[f64]) -> Vec<u8> {
808 let meta = GlobalMetadata::default();
809 let desc = DataObjectDescriptor {
810 obj_type: "ndarray".to_string(),
811 ndim: 1,
812 shape: vec![values.len() as u64],
813 strides: vec![8],
814 dtype: Dtype::Float64,
815 byte_order: ByteOrder::Big,
816 encoding: "none".to_string(),
817 filter: "none".to_string(),
818 compression: "none".to_string(),
819 params: BTreeMap::new(),
820 hash: None,
821 };
822 let data: Vec<u8> = values.iter().flat_map(|v| v.to_be_bytes()).collect();
823 encode(
824 &meta,
825 &[(&desc, data.as_slice())],
826 &EncodeOptions::default(),
827 )
828 .unwrap()
829 }
830
831 fn make_float32_message_le(values: &[f32]) -> Vec<u8> {
832 let meta = GlobalMetadata::default();
833 let desc = DataObjectDescriptor {
834 obj_type: "ndarray".to_string(),
835 ndim: 1,
836 shape: vec![values.len() as u64],
837 strides: vec![4],
838 dtype: Dtype::Float32,
839 byte_order: ByteOrder::Little,
840 encoding: "none".to_string(),
841 filter: "none".to_string(),
842 compression: "none".to_string(),
843 params: BTreeMap::new(),
844 hash: None,
845 };
846 let data: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
847 encode(
848 &meta,
849 &[(&desc, data.as_slice())],
850 &EncodeOptions::default(),
851 )
852 .unwrap()
853 }
854
855 #[test]
856 fn full_mode_valid_float64_passes() {
857 let msg = make_float64_message(&[1.0, 2.0, 3.0, 4.0]);
858 let report = validate_message(&msg, &full_opts());
859 assert!(report.is_ok(), "issues: {:?}", report.issues);
860 }
861
862 #[test]
863 fn full_mode_nan_float64_detected() {
864 let msg = make_float64_message(&[1.0, f64::NAN, 3.0]);
865 let report = validate_message(&msg, &full_opts());
866 assert!(!report.is_ok());
867 let nan_issue = report
868 .issues
869 .iter()
870 .find(|i| i.code == IssueCode::NanDetected);
871 assert!(
872 nan_issue.is_some(),
873 "expected NanDetected, got: {:?}",
874 report.issues
875 );
876 assert!(nan_issue.unwrap().description.contains("element 1"));
877 }
878
879 #[test]
880 fn full_mode_inf_float64_detected() {
881 let msg = make_float64_message(&[1.0, 2.0, f64::INFINITY]);
882 let report = validate_message(&msg, &full_opts());
883 assert!(!report.is_ok());
884 let inf_issue = report
885 .issues
886 .iter()
887 .find(|i| i.code == IssueCode::InfDetected);
888 assert!(
889 inf_issue.is_some(),
890 "expected InfDetected, got: {:?}",
891 report.issues
892 );
893 assert!(inf_issue.unwrap().description.contains("element 2"));
894 }
895
896 #[test]
897 fn full_mode_neg_inf_detected() {
898 let msg = make_float64_message(&[f64::NEG_INFINITY, 1.0]);
899 let report = validate_message(&msg, &full_opts());
900 assert!(!report.is_ok());
901 assert!(
902 report
903 .issues
904 .iter()
905 .any(|i| i.code == IssueCode::InfDetected)
906 );
907 }
908
909 #[test]
910 fn full_mode_float32_le_nan_detected() {
911 let msg = make_float32_message_le(&[1.0, f32::NAN, 3.0]);
912 let report = validate_message(&msg, &full_opts());
913 assert!(!report.is_ok());
914 assert!(
915 report
916 .issues
917 .iter()
918 .any(|i| i.code == IssueCode::NanDetected)
919 );
920 }
921
922 #[test]
923 fn full_mode_float32_le_inf_detected() {
924 let msg = make_float32_message_le(&[f32::INFINITY]);
925 let report = validate_message(&msg, &full_opts());
926 assert!(!report.is_ok());
927 assert!(
928 report
929 .issues
930 .iter()
931 .any(|i| i.code == IssueCode::InfDetected)
932 );
933 }
934
935 #[test]
936 fn full_mode_integer_passes() {
937 let meta = GlobalMetadata::default();
938 let desc = DataObjectDescriptor {
939 obj_type: "ndarray".to_string(),
940 ndim: 1,
941 shape: vec![4],
942 strides: vec![4],
943 dtype: Dtype::Int32,
944 byte_order: ByteOrder::Big,
945 encoding: "none".to_string(),
946 filter: "none".to_string(),
947 compression: "none".to_string(),
948 params: BTreeMap::new(),
949 hash: None,
950 };
951 let data = vec![0u8; 16]; let msg = encode(
953 &meta,
954 &[(&desc, data.as_slice())],
955 &EncodeOptions::default(),
956 )
957 .unwrap();
958 let report = validate_message(&msg, &full_opts());
959 assert!(
960 report.is_ok(),
961 "integer should pass fidelity: {:?}",
962 report.issues
963 );
964 }
965
966 #[test]
967 fn full_mode_hash_verified_false_on_nan() {
968 let msg = make_float64_message(&[f64::NAN]);
969 let report = validate_message(&msg, &full_opts());
970 assert!(
971 !report.hash_verified,
972 "hash_verified should be false when NaN detected"
973 );
974 }
975
976 #[test]
977 fn full_mode_float16_nan() {
978 let meta = GlobalMetadata::default();
979 let desc = DataObjectDescriptor {
980 obj_type: "ndarray".to_string(),
981 ndim: 1,
982 shape: vec![2],
983 strides: vec![2],
984 dtype: Dtype::Float16,
985 byte_order: ByteOrder::Big,
986 encoding: "none".to_string(),
987 filter: "none".to_string(),
988 compression: "none".to_string(),
989 params: BTreeMap::new(),
990 hash: None,
991 };
992 let mut data = vec![0u8; 4]; data[0..2].copy_from_slice(&0x0000u16.to_be_bytes()); data[2..4].copy_from_slice(&0x7C01u16.to_be_bytes()); let msg = encode(
998 &meta,
999 &[(&desc, data.as_slice())],
1000 &EncodeOptions::default(),
1001 )
1002 .unwrap();
1003 let report = validate_message(&msg, &full_opts());
1004 assert!(!report.is_ok());
1005 let nan = report
1006 .issues
1007 .iter()
1008 .find(|i| i.code == IssueCode::NanDetected);
1009 assert!(
1010 nan.is_some(),
1011 "expected float16 NaN, got: {:?}",
1012 report.issues
1013 );
1014 assert!(nan.unwrap().description.contains("element 1"));
1015 }
1016
1017 #[test]
1018 fn full_mode_float16_inf() {
1019 let meta = GlobalMetadata::default();
1020 let desc = DataObjectDescriptor {
1021 obj_type: "ndarray".to_string(),
1022 ndim: 1,
1023 shape: vec![1],
1024 strides: vec![2],
1025 dtype: Dtype::Float16,
1026 byte_order: ByteOrder::Big,
1027 encoding: "none".to_string(),
1028 filter: "none".to_string(),
1029 compression: "none".to_string(),
1030 params: BTreeMap::new(),
1031 hash: None,
1032 };
1033 let data = 0x7C00u16.to_be_bytes().to_vec();
1035 let msg = encode(
1036 &meta,
1037 &[(&desc, data.as_slice())],
1038 &EncodeOptions::default(),
1039 )
1040 .unwrap();
1041 let report = validate_message(&msg, &full_opts());
1042 assert!(!report.is_ok());
1043 assert!(
1044 report
1045 .issues
1046 .iter()
1047 .any(|i| i.code == IssueCode::InfDetected)
1048 );
1049 }
1050
1051 #[test]
1052 fn full_mode_bfloat16_nan() {
1053 let meta = GlobalMetadata::default();
1054 let desc = DataObjectDescriptor {
1055 obj_type: "ndarray".to_string(),
1056 ndim: 1,
1057 shape: vec![1],
1058 strides: vec![2],
1059 dtype: Dtype::Bfloat16,
1060 byte_order: ByteOrder::Big,
1061 encoding: "none".to_string(),
1062 filter: "none".to_string(),
1063 compression: "none".to_string(),
1064 params: BTreeMap::new(),
1065 hash: None,
1066 };
1067 let data = 0x7F81u16.to_be_bytes().to_vec();
1070 let msg = encode(
1071 &meta,
1072 &[(&desc, data.as_slice())],
1073 &EncodeOptions::default(),
1074 )
1075 .unwrap();
1076 let report = validate_message(&msg, &full_opts());
1077 assert!(!report.is_ok());
1078 assert!(
1079 report
1080 .issues
1081 .iter()
1082 .any(|i| i.code == IssueCode::NanDetected)
1083 );
1084 }
1085
1086 #[test]
1087 fn full_mode_complex64_real_nan() {
1088 let meta = GlobalMetadata::default();
1089 let desc = DataObjectDescriptor {
1090 obj_type: "ndarray".to_string(),
1091 ndim: 1,
1092 shape: vec![1],
1093 strides: vec![8],
1094 dtype: Dtype::Complex64,
1095 byte_order: ByteOrder::Big,
1096 encoding: "none".to_string(),
1097 filter: "none".to_string(),
1098 compression: "none".to_string(),
1099 params: BTreeMap::new(),
1100 hash: None,
1101 };
1102 let mut data = Vec::new();
1104 data.extend_from_slice(&f32::NAN.to_be_bytes());
1105 data.extend_from_slice(&0.0f32.to_be_bytes());
1106 let msg = encode(
1107 &meta,
1108 &[(&desc, data.as_slice())],
1109 &EncodeOptions::default(),
1110 )
1111 .unwrap();
1112 let report = validate_message(&msg, &full_opts());
1113 assert!(!report.is_ok());
1114 let nan = report
1115 .issues
1116 .iter()
1117 .find(|i| i.code == IssueCode::NanDetected);
1118 assert!(nan.is_some());
1119 assert!(nan.unwrap().description.contains("real component"));
1120 }
1121
1122 #[test]
1123 fn full_mode_complex128_imag_inf() {
1124 let meta = GlobalMetadata::default();
1125 let desc = DataObjectDescriptor {
1126 obj_type: "ndarray".to_string(),
1127 ndim: 1,
1128 shape: vec![1],
1129 strides: vec![16],
1130 dtype: Dtype::Complex128,
1131 byte_order: ByteOrder::Big,
1132 encoding: "none".to_string(),
1133 filter: "none".to_string(),
1134 compression: "none".to_string(),
1135 params: BTreeMap::new(),
1136 hash: None,
1137 };
1138 let mut data = Vec::new();
1140 data.extend_from_slice(&1.0f64.to_be_bytes());
1141 data.extend_from_slice(&f64::INFINITY.to_be_bytes());
1142 let msg = encode(
1143 &meta,
1144 &[(&desc, data.as_slice())],
1145 &EncodeOptions::default(),
1146 )
1147 .unwrap();
1148 let report = validate_message(&msg, &full_opts());
1149 assert!(!report.is_ok());
1150 let inf = report
1151 .issues
1152 .iter()
1153 .find(|i| i.code == IssueCode::InfDetected);
1154 assert!(inf.is_some());
1155 assert!(inf.unwrap().description.contains("imaginary component"));
1156 }
1157
1158 #[test]
1159 fn full_mode_with_canonical() {
1160 let msg = make_test_message();
1161 let opts = ValidateOptions {
1162 max_level: ValidationLevel::Fidelity,
1163 check_canonical: true,
1164 ..ValidateOptions::default()
1165 };
1166 let report = validate_message(&msg, &opts);
1167 assert!(
1168 report.is_ok(),
1169 "full+canonical should pass: {:?}",
1170 report.issues
1171 );
1172 }
1173
1174 #[test]
1175 fn full_mode_json_serialization() {
1176 let msg = make_float64_message(&[f64::NAN]);
1177 let report = validate_message(&msg, &full_opts());
1178 let json = serde_json::to_string(&report).unwrap();
1179 assert!(json.contains("\"nan_detected\""));
1180 assert!(json.contains("\"fidelity\""));
1181 }
1182
1183 #[test]
1184 fn default_mode_skips_fidelity() {
1185 let msg = make_float64_message(&[f64::NAN]);
1187 let report = validate_message(&msg, &ValidateOptions::default());
1188 assert!(
1190 !report
1191 .issues
1192 .iter()
1193 .any(|i| i.code == IssueCode::NanDetected),
1194 "default mode should not run fidelity: {:?}",
1195 report.issues
1196 );
1197 }
1198
1199 #[test]
1202 fn full_mode_negative_zero_passes() {
1203 let msg = make_float64_message(&[-0.0, 0.0, 1.0]);
1204 let report = validate_message(&msg, &full_opts());
1205 assert!(
1206 report.is_ok(),
1207 "negative zero should pass: {:?}",
1208 report.issues
1209 );
1210 }
1211
1212 #[test]
1213 fn full_mode_subnormal_passes() {
1214 let msg = make_float64_message(&[5e-324, 1.0]);
1216 let report = validate_message(&msg, &full_opts());
1217 assert!(
1218 report.is_ok(),
1219 "subnormals should pass: {:?}",
1220 report.issues
1221 );
1222 }
1223
1224 #[test]
1225 fn full_mode_zero_length_array() {
1226 let meta = GlobalMetadata::default();
1227 let desc = DataObjectDescriptor {
1228 obj_type: "ndarray".to_string(),
1229 ndim: 1,
1230 shape: vec![0],
1231 strides: vec![8],
1232 dtype: Dtype::Float64,
1233 byte_order: ByteOrder::Big,
1234 encoding: "none".to_string(),
1235 filter: "none".to_string(),
1236 compression: "none".to_string(),
1237 params: BTreeMap::new(),
1238 hash: None,
1239 };
1240 let data: Vec<u8> = vec![];
1241 let msg = encode(
1242 &meta,
1243 &[(&desc, data.as_slice())],
1244 &EncodeOptions::default(),
1245 )
1246 .unwrap();
1247 let report = validate_message(&msg, &full_opts());
1248 assert!(
1249 report.is_ok(),
1250 "zero-length array should pass: {:?}",
1251 report.issues
1252 );
1253 }
1254
1255 #[test]
1256 fn full_mode_decoded_size_mismatch() {
1257 let meta = GlobalMetadata::default();
1261 let desc = DataObjectDescriptor {
1262 obj_type: "ndarray".to_string(),
1263 ndim: 1,
1264 shape: vec![3],
1265 strides: vec![4],
1266 dtype: Dtype::Float32,
1267 byte_order: ByteOrder::Big,
1268 encoding: "none".to_string(),
1269 filter: "none".to_string(),
1270 compression: "none".to_string(),
1271 params: BTreeMap::new(),
1272 hash: None,
1273 };
1274 let data = vec![0u8; 12]; let opts = EncodeOptions {
1276 hash_algorithm: None,
1277 ..EncodeOptions::default()
1278 };
1279 let mut msg = encode(&meta, &[(&desc, data.as_slice())], &opts).unwrap();
1280
1281 let mut patched = false;
1287 for i in (0..msg.len() - 1).rev() {
1288 if msg[i] == 0x81 && msg[i + 1] == 0x03 {
1289 msg[i + 1] = 0x04; patched = true;
1291 break;
1292 }
1293 }
1294 assert!(patched, "could not find shape [3] in encoded message");
1295
1296 let report = validate_message(&msg, &full_opts());
1297 let has_mismatch = report
1298 .issues
1299 .iter()
1300 .any(|i| i.code == IssueCode::DecodedSizeMismatch);
1301 assert!(
1302 has_mismatch,
1303 "expected DecodedSizeMismatch, got: {:?}",
1304 report.issues
1305 );
1306 }
1307
1308 #[test]
1309 fn quick_canonical_runs_metadata() {
1310 let msg = make_test_message();
1312 let opts = ValidateOptions {
1313 max_level: ValidationLevel::Structure,
1314 check_canonical: true,
1315 checksum_only: false,
1316 };
1317 let report = validate_message(&msg, &opts);
1318 assert!(report.is_ok(), "issues: {:?}", report.issues);
1320 }
1321
1322 fn build_raw_message(
1329 flags: u16,
1330 frames: &[Vec<u8>], total_length_override: Option<u64>,
1332 streaming: bool,
1333 ) -> Vec<u8> {
1334 use crate::wire::{END_MAGIC, MAGIC};
1335
1336 let mut out = Vec::new();
1337
1338 out.extend_from_slice(MAGIC);
1340 out.extend_from_slice(&2u16.to_be_bytes()); out.extend_from_slice(&flags.to_be_bytes());
1342 out.extend_from_slice(&0u32.to_be_bytes()); out.extend_from_slice(&0u64.to_be_bytes()); for frame in frames {
1347 out.extend_from_slice(frame);
1348 let pad = (8 - (out.len() % 8)) % 8;
1350 out.extend(std::iter::repeat_n(0u8, pad));
1351 }
1352
1353 let ffo = out.len() as u64;
1355 out.extend_from_slice(&ffo.to_be_bytes());
1356 out.extend_from_slice(END_MAGIC);
1357
1358 let total = if streaming { 0u64 } else { out.len() as u64 };
1360 let tl = total_length_override.unwrap_or(total);
1361 out[16..24].copy_from_slice(&tl.to_be_bytes());
1362
1363 out
1364 }
1365
1366 fn build_metadata_frame() -> Vec<u8> {
1368 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1369 let meta = GlobalMetadata::default();
1370 let cbor = crate::metadata::global_metadata_to_cbor(&meta).unwrap();
1371 let total_length = (FRAME_HEADER_SIZE + cbor.len() + FRAME_END.len()) as u64;
1372 let mut frame = Vec::new();
1373 frame.extend_from_slice(FRAME_MAGIC);
1374 frame.extend_from_slice(&1u16.to_be_bytes()); frame.extend_from_slice(&1u16.to_be_bytes()); frame.extend_from_slice(&0u16.to_be_bytes()); frame.extend_from_slice(&total_length.to_be_bytes());
1378 frame.extend_from_slice(&cbor);
1379 frame.extend_from_slice(FRAME_END);
1380 frame
1381 }
1382
1383 fn build_data_object_frame(desc: &DataObjectDescriptor, payload: &[u8]) -> Vec<u8> {
1385 crate::framing::encode_data_object_frame(desc, payload, false).unwrap()
1386 }
1387
1388 fn default_desc() -> DataObjectDescriptor {
1390 DataObjectDescriptor {
1391 obj_type: "ndarray".to_string(),
1392 ndim: 1,
1393 shape: vec![4],
1394 strides: vec![8],
1395 dtype: Dtype::Float64,
1396 byte_order: ByteOrder::Big,
1397 encoding: "none".to_string(),
1398 filter: "none".to_string(),
1399 compression: "none".to_string(),
1400 params: BTreeMap::new(),
1401 hash: None,
1402 }
1403 }
1404
1405 #[test]
1408 fn structure_frame_length_overflow() {
1409 let mut msg = make_test_message();
1412 let frame_start = PREAMBLE_SIZE;
1414 let tl_offset = frame_start + 8;
1416 msg[tl_offset..tl_offset + 8].copy_from_slice(&u64::MAX.to_be_bytes());
1418 let report = validate_message(&msg, &ValidateOptions::default());
1419 assert!(!report.is_ok());
1420 let has_overflow = report
1421 .issues
1422 .iter()
1423 .any(|i| i.code == IssueCode::FrameLengthOverflow);
1424 assert!(
1425 has_overflow,
1426 "expected FrameLengthOverflow, got: {:?}",
1427 report.issues
1428 );
1429 }
1430
1431 #[test]
1434 fn structure_non_zero_padding_between_frames() {
1435 let mut msg = make_test_message();
1438 let frame_start = PREAMBLE_SIZE;
1441 let tl =
1442 u64::from_be_bytes(msg[frame_start + 8..frame_start + 16].try_into().unwrap()) as usize;
1443 let frame_end = frame_start + tl;
1444 let next_aligned = (frame_end + 7) & !7;
1446 if next_aligned > frame_end && next_aligned < msg.len() {
1447 for b in &mut msg[frame_end..next_aligned] {
1449 *b = 0xAA;
1450 }
1451 let report = validate_message(&msg, &ValidateOptions::default());
1452 let has_padding_warn = report
1453 .issues
1454 .iter()
1455 .any(|i| i.code == IssueCode::NonZeroPadding);
1456 assert!(
1457 has_padding_warn,
1458 "expected NonZeroPadding warning, got: {:?}",
1459 report.issues
1460 );
1461 }
1462 else {
1466 }
1469 }
1470
1471 #[test]
1474 fn structure_frame_order_violation() {
1475 let desc = default_desc();
1479 let payload = vec![0u8; 32];
1480 let data_frame = build_data_object_frame(&desc, &payload);
1481 let meta_frame = build_metadata_frame();
1482
1483 let flags = 1u16; let msg = build_raw_message(flags, &[data_frame, meta_frame], None, false);
1486 let report = validate_message(&msg, &ValidateOptions::default());
1487 let has_order = report
1488 .issues
1489 .iter()
1490 .any(|i| i.code == IssueCode::FrameOrderViolation);
1491 assert!(
1492 has_order,
1493 "expected FrameOrderViolation, got: {:?}",
1494 report.issues
1495 );
1496 }
1497
1498 #[test]
1501 fn structure_preceder_not_followed_by_object() {
1502 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1505
1506 let meta = GlobalMetadata {
1507 base: vec![BTreeMap::new()],
1508 ..GlobalMetadata::default()
1509 };
1510 let cbor = crate::metadata::global_metadata_to_cbor(&meta).unwrap();
1511
1512 let total_length = (FRAME_HEADER_SIZE + cbor.len() + FRAME_END.len()) as u64;
1514 let mut preceder_frame = Vec::new();
1515 preceder_frame.extend_from_slice(FRAME_MAGIC);
1516 preceder_frame.extend_from_slice(&8u16.to_be_bytes()); preceder_frame.extend_from_slice(&1u16.to_be_bytes()); preceder_frame.extend_from_slice(&0u16.to_be_bytes()); preceder_frame.extend_from_slice(&total_length.to_be_bytes());
1520 preceder_frame.extend_from_slice(&cbor);
1521 preceder_frame.extend_from_slice(FRAME_END);
1522
1523 let header_meta_frame = build_metadata_frame();
1524
1525 let mut footer_meta_frame = Vec::new();
1533 let footer_cbor =
1534 crate::metadata::global_metadata_to_cbor(&GlobalMetadata::default()).unwrap();
1535 let ftl = (FRAME_HEADER_SIZE + footer_cbor.len() + FRAME_END.len()) as u64;
1536 footer_meta_frame.extend_from_slice(FRAME_MAGIC);
1537 footer_meta_frame.extend_from_slice(&7u16.to_be_bytes()); footer_meta_frame.extend_from_slice(&1u16.to_be_bytes());
1539 footer_meta_frame.extend_from_slice(&0u16.to_be_bytes());
1540 footer_meta_frame.extend_from_slice(&ftl.to_be_bytes());
1541 footer_meta_frame.extend_from_slice(&footer_cbor);
1542 footer_meta_frame.extend_from_slice(FRAME_END);
1543
1544 let flags = (1u16) | (1u16 << 1) | (1u16 << 6); let msg = build_raw_message(
1546 flags,
1547 &[header_meta_frame, preceder_frame, footer_meta_frame],
1548 None,
1549 false,
1550 );
1551 let report = validate_message(&msg, &ValidateOptions::default());
1552 let has_preceder_err = report
1553 .issues
1554 .iter()
1555 .any(|i| i.code == IssueCode::PrecederNotFollowedByObject);
1556 assert!(
1557 has_preceder_err,
1558 "expected PrecederNotFollowedByObject, got: {:?}",
1559 report.issues
1560 );
1561 }
1562
1563 #[test]
1566 fn structure_dangling_preceder() {
1567 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1569
1570 let meta = GlobalMetadata {
1571 base: vec![BTreeMap::new()],
1572 ..GlobalMetadata::default()
1573 };
1574 let cbor = crate::metadata::global_metadata_to_cbor(&meta).unwrap();
1575
1576 let total_length = (FRAME_HEADER_SIZE + cbor.len() + FRAME_END.len()) as u64;
1577 let mut preceder_frame = Vec::new();
1578 preceder_frame.extend_from_slice(FRAME_MAGIC);
1579 preceder_frame.extend_from_slice(&8u16.to_be_bytes());
1580 preceder_frame.extend_from_slice(&1u16.to_be_bytes());
1581 preceder_frame.extend_from_slice(&0u16.to_be_bytes());
1582 preceder_frame.extend_from_slice(&total_length.to_be_bytes());
1583 preceder_frame.extend_from_slice(&cbor);
1584 preceder_frame.extend_from_slice(FRAME_END);
1585
1586 let header_meta_frame = build_metadata_frame();
1587 let flags = 1u16 | (1u16 << 6); let msg = build_raw_message(flags, &[header_meta_frame, preceder_frame], None, false);
1589 let report = validate_message(&msg, &ValidateOptions::default());
1590 let has_dangling = report
1591 .issues
1592 .iter()
1593 .any(|i| i.code == IssueCode::DanglingPreceder);
1594 assert!(
1595 has_dangling,
1596 "expected DanglingPreceder, got: {:?}",
1597 report.issues
1598 );
1599 }
1600
1601 #[test]
1604 fn structure_cbor_before_boundary_unknown() {
1605 use crate::wire::{DATA_OBJECT_FOOTER_SIZE, FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1609
1610 let payload = vec![0u8; 16];
1611 let bad_cbor = vec![0xFF, 0xFF, 0xFF, 0xFF];
1613
1614 let cbor_offset = FRAME_HEADER_SIZE as u64; let body_len = bad_cbor.len() + payload.len() + DATA_OBJECT_FOOTER_SIZE;
1617 let total_length = (FRAME_HEADER_SIZE + body_len) as u64;
1618
1619 let mut frame = Vec::new();
1620 frame.extend_from_slice(FRAME_MAGIC);
1621 frame.extend_from_slice(&4u16.to_be_bytes()); frame.extend_from_slice(&1u16.to_be_bytes()); frame.extend_from_slice(&0u16.to_be_bytes()); frame.extend_from_slice(&total_length.to_be_bytes());
1625 frame.extend_from_slice(&bad_cbor);
1626 frame.extend_from_slice(&payload);
1627 frame.extend_from_slice(&cbor_offset.to_be_bytes());
1628 frame.extend_from_slice(FRAME_END);
1629
1630 let meta_frame = build_metadata_frame();
1631 let flags = 1u16; let msg = build_raw_message(flags, &[meta_frame, frame], None, false);
1633 let report = validate_message(&msg, &ValidateOptions::default());
1634 let has_cbor_err = report
1635 .issues
1636 .iter()
1637 .any(|i| i.code == IssueCode::CborBeforeBoundaryUnknown);
1638 assert!(
1639 has_cbor_err,
1640 "expected CborBeforeBoundaryUnknown, got: {:?}",
1641 report.issues
1642 );
1643 }
1644
1645 #[test]
1648 fn structure_flag_mismatch() {
1649 let mut msg = make_test_message();
1652 let current_flags = u16::from_be_bytes(msg[10..12].try_into().unwrap());
1654 let bad_flags = current_flags | (1u16 << 1); msg[10..12].copy_from_slice(&bad_flags.to_be_bytes());
1658 let report = validate_message(&msg, &ValidateOptions::default());
1659 let has_flag_mismatch = report
1660 .issues
1661 .iter()
1662 .any(|i| i.code == IssueCode::FlagMismatch);
1663 assert!(
1664 has_flag_mismatch,
1665 "expected FlagMismatch, got: {:?}",
1666 report.issues
1667 );
1668 }
1669
1670 #[test]
1673 fn structure_no_metadata_frame() {
1674 let desc = default_desc();
1676 let payload = vec![0u8; 32];
1677 let data_frame = build_data_object_frame(&desc, &payload);
1678
1679 let msg = build_raw_message(0u16, &[data_frame], None, false);
1681 let report = validate_message(&msg, &ValidateOptions::default());
1682 let has_no_meta = report
1683 .issues
1684 .iter()
1685 .any(|i| i.code == IssueCode::NoMetadataFrame);
1686 assert!(
1687 has_no_meta,
1688 "expected NoMetadataFrame, got: {:?}",
1689 report.issues
1690 );
1691 }
1692
1693 #[test]
1696 fn structure_streaming_mode_validates() {
1697 let meta_frame = build_metadata_frame();
1699 let desc = default_desc();
1700 let payload = vec![0u8; 32];
1701 let data_frame = build_data_object_frame(&desc, &payload);
1702 let flags = 1u16; let msg = build_raw_message(flags, &[meta_frame, data_frame], None, true);
1704 let report = validate_message(&msg, &ValidateOptions::default());
1705 let has_fatal = report.issues.iter().any(|i| {
1708 i.severity == IssueSeverity::Error
1709 && !matches!(
1710 i.code,
1711 IssueCode::FlagMismatch
1712 | IssueCode::FooterOffsetMismatch
1713 | IssueCode::NoMetadataFrame
1714 )
1715 });
1716 assert!(
1718 !has_fatal,
1719 "unexpected fatal error in streaming mode: {:?}",
1720 report.issues
1721 );
1722 }
1723
1724 #[test]
1725 fn structure_streaming_mode_bad_postamble() {
1726 let meta_frame = build_metadata_frame();
1728 let flags = 1u16;
1729 let mut msg = build_raw_message(flags, &[meta_frame], None, true);
1730 let end = msg.len();
1732 msg[end - 8..end].copy_from_slice(b"BADMAGIC");
1733 let report = validate_message(&msg, &ValidateOptions::default());
1734 assert!(!report.is_ok());
1735 let has_postamble = report
1736 .issues
1737 .iter()
1738 .any(|i| i.code == IssueCode::PostambleInvalid);
1739 assert!(
1740 has_postamble,
1741 "expected PostambleInvalid in streaming mode, got: {:?}",
1742 report.issues
1743 );
1744 }
1745
1746 fn make_message_with_patched_descriptor(patch: impl FnOnce(&mut ciborium::Value)) -> Vec<u8> {
1754 let meta = GlobalMetadata::default();
1756 let desc = default_desc();
1757 let data = vec![0u8; 32];
1758 let opts = EncodeOptions {
1759 hash_algorithm: None,
1760 ..EncodeOptions::default()
1761 };
1762 let _msg = encode(&meta, &[(&desc, data.as_slice())], &opts).unwrap();
1763
1764 let cbor_bytes = crate::metadata::object_descriptor_to_cbor(&desc).unwrap();
1766 let mut value: ciborium::Value = ciborium::from_reader(cbor_bytes.as_slice()).unwrap();
1767 patch(&mut value);
1768 let mut patched_cbor = Vec::new();
1769 ciborium::into_writer(&value, &mut patched_cbor).unwrap();
1770
1771 use crate::wire::{
1773 DATA_OBJECT_FOOTER_SIZE, DataObjectFlags, FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC,
1774 };
1775 let payload = vec![0u8; 32];
1776 let cbor_offset = (FRAME_HEADER_SIZE + payload.len()) as u64;
1777 let total_length =
1778 (FRAME_HEADER_SIZE + payload.len() + patched_cbor.len() + DATA_OBJECT_FOOTER_SIZE)
1779 as u64;
1780
1781 let mut frame = Vec::new();
1782 frame.extend_from_slice(FRAME_MAGIC);
1783 frame.extend_from_slice(&4u16.to_be_bytes()); frame.extend_from_slice(&1u16.to_be_bytes()); frame.extend_from_slice(&DataObjectFlags::CBOR_AFTER_PAYLOAD.to_be_bytes()); frame.extend_from_slice(&total_length.to_be_bytes());
1787 frame.extend_from_slice(&payload);
1788 frame.extend_from_slice(&patched_cbor);
1789 frame.extend_from_slice(&cbor_offset.to_be_bytes());
1790 frame.extend_from_slice(FRAME_END);
1791
1792 let meta_frame = build_metadata_frame();
1793 let flags = 1u16; build_raw_message(flags, &[meta_frame, frame], None, false)
1795 }
1796
1797 fn cbor_map_set(value: &mut ciborium::Value, key: &str, new_val: ciborium::Value) {
1799 if let ciborium::Value::Map(pairs) = value {
1800 for (k, v) in pairs.iter_mut() {
1801 if let ciborium::Value::Text(s) = k
1802 && s == key
1803 {
1804 *v = new_val;
1805 return;
1806 }
1807 }
1808 pairs.push((ciborium::Value::Text(key.to_string()), new_val));
1810 }
1811 }
1812
1813 #[test]
1816 fn metadata_index_count_mismatch() {
1817 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1820
1821 let desc = default_desc();
1822 let payload = vec![0u8; 32];
1823 let data_frame = build_data_object_frame(&desc, &payload);
1824 let meta_frame = build_metadata_frame();
1825
1826 let idx = crate::types::IndexFrame {
1828 object_count: 5,
1829 offsets: vec![0, 100, 200, 300, 400],
1830 lengths: vec![50, 50, 50, 50, 50],
1831 };
1832 let idx_cbor = crate::metadata::index_to_cbor(&idx).unwrap();
1833 let idx_total = (FRAME_HEADER_SIZE + idx_cbor.len() + FRAME_END.len()) as u64;
1834 let mut idx_frame = Vec::new();
1835 idx_frame.extend_from_slice(FRAME_MAGIC);
1836 idx_frame.extend_from_slice(&2u16.to_be_bytes()); idx_frame.extend_from_slice(&1u16.to_be_bytes());
1838 idx_frame.extend_from_slice(&0u16.to_be_bytes());
1839 idx_frame.extend_from_slice(&idx_total.to_be_bytes());
1840 idx_frame.extend_from_slice(&idx_cbor);
1841 idx_frame.extend_from_slice(FRAME_END);
1842
1843 let flags = 1u16 | (1u16 << 2); let msg = build_raw_message(flags, &[meta_frame, idx_frame, data_frame], None, false);
1845 let report = validate_message(&msg, &ValidateOptions::default());
1846 let has_idx_mismatch = report
1847 .issues
1848 .iter()
1849 .any(|i| i.code == IssueCode::IndexCountMismatch);
1850 assert!(
1851 has_idx_mismatch,
1852 "expected IndexCountMismatch, got: {:?}",
1853 report.issues
1854 );
1855 }
1856
1857 #[test]
1860 fn metadata_index_offset_mismatch() {
1861 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
1863
1864 let desc = default_desc();
1865 let payload = vec![0u8; 32];
1866 let data_frame = build_data_object_frame(&desc, &payload);
1867 let meta_frame = build_metadata_frame();
1868
1869 let idx = crate::types::IndexFrame {
1870 object_count: 1,
1871 offsets: vec![9999], lengths: vec![50],
1873 };
1874 let idx_cbor = crate::metadata::index_to_cbor(&idx).unwrap();
1875 let idx_total = (FRAME_HEADER_SIZE + idx_cbor.len() + FRAME_END.len()) as u64;
1876 let mut idx_frame = Vec::new();
1877 idx_frame.extend_from_slice(FRAME_MAGIC);
1878 idx_frame.extend_from_slice(&2u16.to_be_bytes()); idx_frame.extend_from_slice(&1u16.to_be_bytes());
1880 idx_frame.extend_from_slice(&0u16.to_be_bytes());
1881 idx_frame.extend_from_slice(&idx_total.to_be_bytes());
1882 idx_frame.extend_from_slice(&idx_cbor);
1883 idx_frame.extend_from_slice(FRAME_END);
1884
1885 let flags = 1u16 | (1u16 << 2); let msg = build_raw_message(flags, &[meta_frame, idx_frame, data_frame], None, false);
1887 let report = validate_message(&msg, &ValidateOptions::default());
1888 let has_offset_mismatch = report
1889 .issues
1890 .iter()
1891 .any(|i| i.code == IssueCode::IndexOffsetMismatch);
1892 assert!(
1893 has_offset_mismatch,
1894 "expected IndexOffsetMismatch, got: {:?}",
1895 report.issues
1896 );
1897 }
1898
1899 #[test]
1902 fn metadata_unknown_encoding() {
1903 let msg = make_message_with_patched_descriptor(|v| {
1904 cbor_map_set(
1905 v,
1906 "encoding",
1907 ciborium::Value::Text("turbo_zip".to_string()),
1908 );
1909 });
1910 let report = validate_message(&msg, &ValidateOptions::default());
1911 let has_unk = report
1912 .issues
1913 .iter()
1914 .any(|i| i.code == IssueCode::UnknownEncoding);
1915 assert!(
1916 has_unk,
1917 "expected UnknownEncoding, got: {:?}",
1918 report.issues
1919 );
1920 }
1921
1922 #[test]
1925 fn metadata_unknown_filter() {
1926 let msg = make_message_with_patched_descriptor(|v| {
1927 cbor_map_set(
1928 v,
1929 "filter",
1930 ciborium::Value::Text("mega_filter".to_string()),
1931 );
1932 });
1933 let report = validate_message(&msg, &ValidateOptions::default());
1934 let has_unk = report
1935 .issues
1936 .iter()
1937 .any(|i| i.code == IssueCode::UnknownFilter);
1938 assert!(has_unk, "expected UnknownFilter, got: {:?}", report.issues);
1939 }
1940
1941 #[test]
1944 fn metadata_unknown_compression() {
1945 let msg = make_message_with_patched_descriptor(|v| {
1946 cbor_map_set(
1947 v,
1948 "compression",
1949 ciborium::Value::Text("snappy9000".to_string()),
1950 );
1951 });
1952 let report = validate_message(&msg, &ValidateOptions::default());
1953 let has_unk = report
1954 .issues
1955 .iter()
1956 .any(|i| i.code == IssueCode::UnknownCompression);
1957 assert!(
1958 has_unk,
1959 "expected UnknownCompression, got: {:?}",
1960 report.issues
1961 );
1962 }
1963
1964 #[test]
1967 fn metadata_empty_obj_type() {
1968 let msg = make_message_with_patched_descriptor(|v| {
1969 cbor_map_set(v, "type", ciborium::Value::Text(String::new()));
1970 });
1971 let report = validate_message(&msg, &ValidateOptions::default());
1972 let has_empty = report
1973 .issues
1974 .iter()
1975 .any(|i| i.code == IssueCode::EmptyObjType);
1976 assert!(has_empty, "expected EmptyObjType, got: {:?}", report.issues);
1977 }
1978
1979 #[test]
1982 fn metadata_ndim_shape_mismatch() {
1983 let msg = make_message_with_patched_descriptor(|v| {
1984 cbor_map_set(v, "ndim", ciborium::Value::Integer(3.into()));
1986 });
1987 let report = validate_message(&msg, &ValidateOptions::default());
1988 let has_ndim = report
1989 .issues
1990 .iter()
1991 .any(|i| i.code == IssueCode::NdimShapeMismatch);
1992 assert!(
1993 has_ndim,
1994 "expected NdimShapeMismatch, got: {:?}",
1995 report.issues
1996 );
1997 }
1998
1999 #[test]
2002 fn metadata_strides_shape_mismatch() {
2003 let msg = make_message_with_patched_descriptor(|v| {
2004 cbor_map_set(
2006 v,
2007 "strides",
2008 ciborium::Value::Array(vec![
2009 ciborium::Value::Integer(8.into()),
2010 ciborium::Value::Integer(4.into()),
2011 ]),
2012 );
2013 });
2014 let report = validate_message(&msg, &ValidateOptions::default());
2015 let has_strides = report
2016 .issues
2017 .iter()
2018 .any(|i| i.code == IssueCode::StridesShapeMismatch);
2019 assert!(
2020 has_strides,
2021 "expected StridesShapeMismatch, got: {:?}",
2022 report.issues
2023 );
2024 }
2025
2026 #[test]
2029 fn metadata_shape_overflow() {
2030 let msg = make_message_with_patched_descriptor(|v| {
2031 cbor_map_set(v, "ndim", ciborium::Value::Integer(2.into()));
2033 cbor_map_set(
2034 v,
2035 "shape",
2036 ciborium::Value::Array(vec![
2037 ciborium::Value::Integer(u64::MAX.into()),
2038 ciborium::Value::Integer(2.into()),
2039 ]),
2040 );
2041 cbor_map_set(
2042 v,
2043 "strides",
2044 ciborium::Value::Array(vec![
2045 ciborium::Value::Integer(8.into()),
2046 ciborium::Value::Integer(8.into()),
2047 ]),
2048 );
2049 });
2050 let report = validate_message(&msg, &ValidateOptions::default());
2051 let has_overflow = report
2052 .issues
2053 .iter()
2054 .any(|i| i.code == IssueCode::ShapeOverflow);
2055 assert!(
2056 has_overflow,
2057 "expected ShapeOverflow, got: {:?}",
2058 report.issues
2059 );
2060 }
2061
2062 #[test]
2065 fn metadata_reserved_not_a_map() {
2066 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2070
2071 let mut base_entry = BTreeMap::new();
2073 base_entry.insert(
2074 "_reserved_".to_string(),
2075 ciborium::Value::Text("not_a_map".to_string()),
2076 );
2077 let meta = GlobalMetadata {
2078 base: vec![base_entry],
2079 ..GlobalMetadata::default()
2080 };
2081 let meta_cbor = crate::metadata::global_metadata_to_cbor(&meta).unwrap();
2082 let total_length = (FRAME_HEADER_SIZE + meta_cbor.len() + FRAME_END.len()) as u64;
2083 let mut meta_frame = Vec::new();
2084 meta_frame.extend_from_slice(FRAME_MAGIC);
2085 meta_frame.extend_from_slice(&1u16.to_be_bytes()); meta_frame.extend_from_slice(&1u16.to_be_bytes());
2087 meta_frame.extend_from_slice(&0u16.to_be_bytes());
2088 meta_frame.extend_from_slice(&total_length.to_be_bytes());
2089 meta_frame.extend_from_slice(&meta_cbor);
2090 meta_frame.extend_from_slice(FRAME_END);
2091
2092 let desc = default_desc();
2093 let payload = vec![0u8; 32];
2094 let data_frame = build_data_object_frame(&desc, &payload);
2095 let flags = 1u16; let msg = build_raw_message(flags, &[meta_frame, data_frame], None, false);
2097 let report = validate_message(&msg, &ValidateOptions::default());
2098 let has_reserved_err = report
2099 .issues
2100 .iter()
2101 .any(|i| i.code == IssueCode::ReservedNotAMap);
2102 assert!(
2103 has_reserved_err,
2104 "expected ReservedNotAMap, got: {:?}",
2105 report.issues
2106 );
2107 }
2108
2109 #[test]
2112 fn metadata_hash_frame_cbor_parse_failed() {
2113 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2115
2116 let meta_frame = build_metadata_frame();
2117 let desc = default_desc();
2118 let payload = vec![0u8; 32];
2119 let data_frame = build_data_object_frame(&desc, &payload);
2120
2121 let garbage_cbor = vec![0xFF, 0xFF, 0xFF, 0xFF];
2123 let hash_total = (FRAME_HEADER_SIZE + garbage_cbor.len() + FRAME_END.len()) as u64;
2124 let mut hash_frame = Vec::new();
2125 hash_frame.extend_from_slice(FRAME_MAGIC);
2126 hash_frame.extend_from_slice(&3u16.to_be_bytes()); hash_frame.extend_from_slice(&1u16.to_be_bytes());
2128 hash_frame.extend_from_slice(&0u16.to_be_bytes());
2129 hash_frame.extend_from_slice(&hash_total.to_be_bytes());
2130 hash_frame.extend_from_slice(&garbage_cbor);
2131 hash_frame.extend_from_slice(FRAME_END);
2132
2133 let flags = 1u16 | (1u16 << 4); let msg = build_raw_message(flags, &[meta_frame, hash_frame, data_frame], None, false);
2135 let report = validate_message(&msg, &ValidateOptions::default());
2136 let has_hash_err = report
2137 .issues
2138 .iter()
2139 .any(|i| i.code == IssueCode::HashFrameCborParseFailed);
2140 assert!(
2141 has_hash_err,
2142 "expected HashFrameCborParseFailed, got: {:?}",
2143 report.issues
2144 );
2145 }
2146
2147 #[test]
2150 fn metadata_preceder_base_count_wrong() {
2151 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2154
2155 let prec_meta = GlobalMetadata {
2156 base: vec![BTreeMap::new(), BTreeMap::new()], ..GlobalMetadata::default()
2158 };
2159 let prec_cbor = crate::metadata::global_metadata_to_cbor(&prec_meta).unwrap();
2160 let prec_total = (FRAME_HEADER_SIZE + prec_cbor.len() + FRAME_END.len()) as u64;
2161 let mut preceder_frame = Vec::new();
2162 preceder_frame.extend_from_slice(FRAME_MAGIC);
2163 preceder_frame.extend_from_slice(&8u16.to_be_bytes()); preceder_frame.extend_from_slice(&1u16.to_be_bytes());
2165 preceder_frame.extend_from_slice(&0u16.to_be_bytes());
2166 preceder_frame.extend_from_slice(&prec_total.to_be_bytes());
2167 preceder_frame.extend_from_slice(&prec_cbor);
2168 preceder_frame.extend_from_slice(FRAME_END);
2169
2170 let meta_frame = build_metadata_frame();
2171 let desc = default_desc();
2172 let payload = vec![0u8; 32];
2173 let data_frame = build_data_object_frame(&desc, &payload);
2174
2175 let flags = 1u16 | (1u16 << 6); let msg = build_raw_message(
2177 flags,
2178 &[meta_frame, preceder_frame, data_frame],
2179 None,
2180 false,
2181 );
2182 let report = validate_message(&msg, &ValidateOptions::default());
2183 let has_prec_err = report
2184 .issues
2185 .iter()
2186 .any(|i| i.code == IssueCode::PrecederBaseCountWrong);
2187 assert!(
2188 has_prec_err,
2189 "expected PrecederBaseCountWrong, got: {:?}",
2190 report.issues
2191 );
2192 }
2193
2194 #[test]
2201 fn integrity_hash_frame_fallback_verified() {
2202 let msg = make_test_message();
2206 let report = validate_message(&msg, &ValidateOptions::default());
2207 assert!(report.hash_verified, "issues: {:?}", report.issues);
2208 }
2209
2210 #[test]
2213 fn integrity_unknown_hash_algorithm() {
2214 let msg = make_message_with_patched_descriptor(|v| {
2216 let hash_map = ciborium::Value::Map(vec![
2217 (
2218 ciborium::Value::Text("type".to_string()),
2219 ciborium::Value::Text("sha9001".to_string()),
2220 ),
2221 (
2222 ciborium::Value::Text("value".to_string()),
2223 ciborium::Value::Text("deadbeef".to_string()),
2224 ),
2225 ]);
2226 cbor_map_set(v, "hash", hash_map);
2227 });
2228 let report = validate_message(&msg, &ValidateOptions::default());
2229 let has_unk_hash = report
2230 .issues
2231 .iter()
2232 .any(|i| i.code == IssueCode::UnknownHashAlgorithm);
2233 assert!(
2234 has_unk_hash,
2235 "expected UnknownHashAlgorithm, got: {:?}",
2236 report.issues
2237 );
2238 }
2239
2240 #[test]
2243 fn integrity_decode_pipeline_failed_corrupt_compressed() {
2244 #[cfg(feature = "zstd")]
2247 {
2248 let meta = GlobalMetadata::default();
2249 let desc = DataObjectDescriptor {
2250 obj_type: "ndarray".to_string(),
2251 ndim: 1,
2252 shape: vec![4],
2253 strides: vec![8],
2254 dtype: Dtype::Float64,
2255 byte_order: ByteOrder::Big,
2256 encoding: "none".to_string(),
2257 filter: "none".to_string(),
2258 compression: "zstd".to_string(),
2259 params: BTreeMap::new(),
2260 hash: None,
2261 };
2262 let data = vec![0u8; 32];
2263 let opts = EncodeOptions {
2264 hash_algorithm: None,
2265 ..EncodeOptions::default()
2266 };
2267 let mut msg = encode(&meta, &[(&desc, data.as_slice())], &opts).unwrap();
2268 let pa_start = msg.len() - crate::wire::POSTAMBLE_SIZE;
2271 let target = pa_start * 3 / 4;
2273 if target < msg.len() {
2274 msg[target] ^= 0xFF;
2275 msg[target.saturating_sub(1)] ^= 0xFF;
2276 }
2277 let report = validate_message(&msg, &ValidateOptions::default());
2278 let has_pipeline_err = report.issues.iter().any(|i| {
2279 matches!(
2280 i.code,
2281 IssueCode::DecodePipelineFailed | IssueCode::HashMismatch
2282 )
2283 });
2284 assert!(
2285 has_pipeline_err || !report.is_ok(),
2286 "expected DecodePipelineFailed or error, got: {:?}",
2287 report.issues
2288 );
2289 }
2290 }
2291
2292 #[test]
2295 fn integrity_shape_product_overflow_pipeline() {
2296 let msg = make_message_with_patched_descriptor(|v| {
2299 cbor_map_set(v, "compression", ciborium::Value::Text("zstd".to_string()));
2300 cbor_map_set(v, "ndim", ciborium::Value::Integer(2.into()));
2301 cbor_map_set(
2302 v,
2303 "shape",
2304 ciborium::Value::Array(vec![
2305 ciborium::Value::Integer(u64::MAX.into()),
2306 ciborium::Value::Integer(2.into()),
2307 ]),
2308 );
2309 cbor_map_set(
2310 v,
2311 "strides",
2312 ciborium::Value::Array(vec![
2313 ciborium::Value::Integer(8.into()),
2314 ciborium::Value::Integer(8.into()),
2315 ]),
2316 );
2317 });
2318 let report = validate_message(&msg, &ValidateOptions::default());
2319 let has_shape_or_pipeline = report.issues.iter().any(|i| {
2321 matches!(
2322 i.code,
2323 IssueCode::ShapeOverflow | IssueCode::PipelineConfigFailed
2324 )
2325 });
2326 assert!(
2327 has_shape_or_pipeline,
2328 "expected ShapeOverflow or PipelineConfigFailed, got: {:?}",
2329 report.issues
2330 );
2331 }
2332
2333 #[test]
2336 fn integrity_descriptor_reparse_without_metadata() {
2337 let msg = make_test_message();
2340 let opts = ValidateOptions {
2341 max_level: ValidationLevel::Integrity,
2342 checksum_only: true, check_canonical: false,
2344 };
2345 let report = validate_message(&msg, &opts);
2346 assert!(
2348 report.hash_verified,
2349 "expected hash_verified in checksum mode, got: {:?}",
2350 report.issues
2351 );
2352 }
2353
2354 #[test]
2361 fn fidelity_bitmask_valid() {
2362 let meta = GlobalMetadata::default();
2364 let desc = DataObjectDescriptor {
2365 obj_type: "ndarray".to_string(),
2366 ndim: 1,
2367 shape: vec![16],
2368 strides: vec![1],
2369 dtype: Dtype::Bitmask,
2370 byte_order: ByteOrder::Big,
2371 encoding: "none".to_string(),
2372 filter: "none".to_string(),
2373 compression: "none".to_string(),
2374 params: BTreeMap::new(),
2375 hash: None,
2376 };
2377 let data = vec![0u8; 2]; let msg = encode(
2379 &meta,
2380 &[(&desc, data.as_slice())],
2381 &EncodeOptions::default(),
2382 )
2383 .unwrap();
2384 let report = validate_message(&msg, &full_opts());
2385 assert!(
2386 report.is_ok(),
2387 "bitmask should pass fidelity: {:?}",
2388 report.issues
2389 );
2390 }
2391
2392 #[test]
2393 fn fidelity_bitmask_non_byte_aligned() {
2394 let meta = GlobalMetadata::default();
2396 let desc = DataObjectDescriptor {
2397 obj_type: "ndarray".to_string(),
2398 ndim: 1,
2399 shape: vec![13],
2400 strides: vec![1],
2401 dtype: Dtype::Bitmask,
2402 byte_order: ByteOrder::Big,
2403 encoding: "none".to_string(),
2404 filter: "none".to_string(),
2405 compression: "none".to_string(),
2406 params: BTreeMap::new(),
2407 hash: None,
2408 };
2409 let data = vec![0u8; 2]; let msg = encode(
2411 &meta,
2412 &[(&desc, data.as_slice())],
2413 &EncodeOptions::default(),
2414 )
2415 .unwrap();
2416 let report = validate_message(&msg, &full_opts());
2417 assert!(
2418 report.is_ok(),
2419 "bitmask (non-byte-aligned) should pass fidelity: {:?}",
2420 report.issues
2421 );
2422 }
2423
2424 #[test]
2427 fn fidelity_decoded_size_overflow() {
2428 let msg = make_message_with_patched_descriptor(|v| {
2432 cbor_map_set(v, "ndim", ciborium::Value::Integer(1.into()));
2436 let big = (u64::MAX / 8) + 1;
2438 cbor_map_set(
2439 v,
2440 "shape",
2441 ciborium::Value::Array(vec![ciborium::Value::Integer(big.into())]),
2442 );
2443 cbor_map_set(
2444 v,
2445 "strides",
2446 ciborium::Value::Array(vec![ciborium::Value::Integer(8.into())]),
2447 );
2448 });
2449 let report = validate_message(&msg, &full_opts());
2450 let has_size_issue = report.issues.iter().any(|i| {
2451 matches!(
2452 i.code,
2453 IssueCode::DecodedSizeMismatch | IssueCode::ShapeOverflow
2454 )
2455 });
2456 assert!(
2457 has_size_issue,
2458 "expected DecodedSizeMismatch or ShapeOverflow, got: {:?}",
2459 report.issues
2460 );
2461 }
2462
2463 #[test]
2466 fn integrity_no_hash_available() {
2467 let meta = GlobalMetadata::default();
2469 let desc = default_desc();
2470 let data = vec![0u8; 32];
2471 let opts = EncodeOptions {
2472 hash_algorithm: None,
2473 ..EncodeOptions::default()
2474 };
2475 let msg = encode(&meta, &[(&desc, data.as_slice())], &opts).unwrap();
2476 let report = validate_message(&msg, &ValidateOptions::default());
2477 let has_no_hash = report
2478 .issues
2479 .iter()
2480 .any(|i| i.code == IssueCode::NoHashAvailable);
2481 assert!(
2482 has_no_hash,
2483 "expected NoHashAvailable, got: {:?}",
2484 report.issues
2485 );
2486 assert!(!report.hash_verified);
2487 }
2488
2489 #[test]
2492 fn structure_total_length_overflow() {
2493 let mut msg = make_test_message();
2497 msg[16..24].copy_from_slice(&u64::MAX.to_be_bytes());
2498 let report = validate_message(&msg, &ValidateOptions::default());
2499 assert!(!report.is_ok());
2500 let has_err = report.issues.iter().any(|i| {
2503 matches!(
2504 i.code,
2505 IssueCode::TotalLengthOverflow | IssueCode::TotalLengthExceedsBuffer
2506 )
2507 });
2508 assert!(has_err, "expected length error, got: {:?}", report.issues);
2509 }
2510
2511 #[test]
2514 fn metadata_hash_frame_count_mismatch() {
2515 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2517
2518 let meta_frame = build_metadata_frame();
2519 let desc = default_desc();
2520 let payload = vec![0u8; 32];
2521 let data_frame = build_data_object_frame(&desc, &payload);
2522
2523 let hash_frame_data = crate::types::HashFrame {
2524 object_count: 3,
2525 hash_type: "xxh3".to_string(),
2526 hashes: vec!["aaa".to_string(), "bbb".to_string(), "ccc".to_string()],
2527 };
2528 let hash_cbor = crate::metadata::hash_frame_to_cbor(&hash_frame_data).unwrap();
2529 let hash_total = (FRAME_HEADER_SIZE + hash_cbor.len() + FRAME_END.len()) as u64;
2530 let mut hash_frame = Vec::new();
2531 hash_frame.extend_from_slice(FRAME_MAGIC);
2532 hash_frame.extend_from_slice(&3u16.to_be_bytes()); hash_frame.extend_from_slice(&1u16.to_be_bytes());
2534 hash_frame.extend_from_slice(&0u16.to_be_bytes());
2535 hash_frame.extend_from_slice(&hash_total.to_be_bytes());
2536 hash_frame.extend_from_slice(&hash_cbor);
2537 hash_frame.extend_from_slice(FRAME_END);
2538
2539 let flags = 1u16 | (1u16 << 4); let msg = build_raw_message(flags, &[meta_frame, hash_frame, data_frame], None, false);
2541 let report = validate_message(&msg, &ValidateOptions::default());
2542 let has_count_mismatch = report
2543 .issues
2544 .iter()
2545 .any(|i| i.code == IssueCode::HashFrameCountMismatch);
2546 assert!(
2547 has_count_mismatch,
2548 "expected HashFrameCountMismatch, got: {:?}",
2549 report.issues
2550 );
2551 }
2552
2553 #[test]
2556 fn fidelity_raw_payload_scan() {
2557 let msg = make_float64_message(&[1.0, 2.0, 3.0, 4.0]);
2561 let report = validate_message(&msg, &full_opts());
2562 assert!(report.is_ok(), "issues: {:?}", report.issues);
2563 }
2564
2565 #[test]
2570 fn structure_streaming_ffo_high() {
2571 let meta_frame = build_metadata_frame();
2572 let desc = default_desc();
2573 let payload = vec![0u8; 32];
2574 let data_frame = build_data_object_frame(&desc, &payload);
2575 let flags = 1u16;
2576 let mut msg = build_raw_message(flags, &[meta_frame, data_frame], None, true);
2577 let pa_start = msg.len() - 16;
2578 let bad_ffo: u64 = (msg.len() + 100) as u64;
2579 msg[pa_start..pa_start + 8].copy_from_slice(&bad_ffo.to_be_bytes());
2580 let report = validate_message(&msg, &ValidateOptions::default());
2581 assert!(
2582 report
2583 .issues
2584 .iter()
2585 .any(|i| i.code == IssueCode::FooterOffsetOutOfRange),
2586 "expected FooterOffsetOutOfRange, got: {:?}",
2587 report.issues
2588 );
2589 }
2590
2591 #[test]
2592 fn structure_double_preceder() {
2593 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2594 let meta_cbor =
2595 crate::metadata::global_metadata_to_cbor(&GlobalMetadata::default()).unwrap();
2596 let total_length = (FRAME_HEADER_SIZE + meta_cbor.len() + FRAME_END.len()) as u64;
2597 let mut pf = Vec::new();
2598 pf.extend_from_slice(FRAME_MAGIC);
2599 pf.extend_from_slice(&8u16.to_be_bytes());
2600 pf.extend_from_slice(&1u16.to_be_bytes());
2601 pf.extend_from_slice(&0u16.to_be_bytes());
2602 pf.extend_from_slice(&total_length.to_be_bytes());
2603 pf.extend_from_slice(&meta_cbor);
2604 pf.extend_from_slice(FRAME_END);
2605 let pf2 = pf.clone();
2606 let desc = default_desc();
2607 let data_frame = build_data_object_frame(&desc, &[0u8; 32]);
2608 let hm = build_metadata_frame();
2609 let flags = 1u16 | (1u16 << 6);
2610 let msg = build_raw_message(flags, &[hm, pf, pf2, data_frame], None, false);
2611 let report = validate_message(&msg, &ValidateOptions::default());
2612 assert!(
2613 report
2614 .issues
2615 .iter()
2616 .any(|i| i.code == IssueCode::PrecederNotFollowedByObject),
2617 "expected PrecederNotFollowedByObject, got: {:?}",
2618 report.issues
2619 );
2620 }
2621
2622 #[test]
2623 fn structure_trailing_preceder() {
2624 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2625 let meta_cbor =
2626 crate::metadata::global_metadata_to_cbor(&GlobalMetadata::default()).unwrap();
2627 let tl = (FRAME_HEADER_SIZE + meta_cbor.len() + FRAME_END.len()) as u64;
2628 let mut pf = Vec::new();
2629 pf.extend_from_slice(FRAME_MAGIC);
2630 pf.extend_from_slice(&8u16.to_be_bytes());
2631 pf.extend_from_slice(&1u16.to_be_bytes());
2632 pf.extend_from_slice(&0u16.to_be_bytes());
2633 pf.extend_from_slice(&tl.to_be_bytes());
2634 pf.extend_from_slice(&meta_cbor);
2635 pf.extend_from_slice(FRAME_END);
2636 let hm = build_metadata_frame();
2637 let desc = default_desc();
2638 let df = build_data_object_frame(&desc, &[0u8; 32]);
2639 let flags = 1u16 | (1u16 << 6);
2640 let msg = build_raw_message(flags, &[hm, df, pf], None, false);
2641 let report = validate_message(&msg, &ValidateOptions::default());
2642 assert!(
2643 report
2644 .issues
2645 .iter()
2646 .any(|i| i.code == IssueCode::DanglingPreceder),
2647 "expected DanglingPreceder, got: {:?}",
2648 report.issues
2649 );
2650 }
2651
2652 #[test]
2653 fn structure_flag_mismatch_extra() {
2654 let meta_frame = build_metadata_frame();
2655 let desc = default_desc();
2656 let data_frame = build_data_object_frame(&desc, &[0u8; 32]);
2657 let msg = build_raw_message(0u16, &[meta_frame, data_frame], None, false);
2658 let report = validate_message(&msg, &ValidateOptions::default());
2659 assert!(
2660 report
2661 .issues
2662 .iter()
2663 .any(|i| i.code == IssueCode::FlagMismatch),
2664 "expected FlagMismatch, got: {:?}",
2665 report.issues
2666 );
2667 }
2668
2669 #[test]
2674 fn fidelity_bitmask_wrong_size() {
2675 let msg = make_message_with_patched_descriptor(|v| {
2676 cbor_map_set(v, "dtype", ciborium::Value::Text("bitmask".to_string()));
2677 cbor_map_set(v, "ndim", ciborium::Value::Integer(1.into()));
2678 cbor_map_set(
2679 v,
2680 "shape",
2681 ciborium::Value::Array(vec![ciborium::Value::Integer(1000.into())]),
2682 );
2683 cbor_map_set(
2684 v,
2685 "strides",
2686 ciborium::Value::Array(vec![ciborium::Value::Integer(1.into())]),
2687 );
2688 });
2689 let report = validate_message(&msg, &full_opts());
2690 assert!(
2691 report
2692 .issues
2693 .iter()
2694 .any(|i| i.code == IssueCode::DecodedSizeMismatch),
2695 "expected DecodedSizeMismatch for bitmask, got: {:?}",
2696 report.issues
2697 );
2698 }
2699
2700 #[test]
2701 fn fidelity_multi_object_mixed() {
2702 let meta = GlobalMetadata::default();
2703 let desc = DataObjectDescriptor {
2704 obj_type: "ndarray".to_string(),
2705 ndim: 1,
2706 shape: vec![2],
2707 strides: vec![8],
2708 dtype: Dtype::Float64,
2709 byte_order: ByteOrder::Big,
2710 encoding: "none".to_string(),
2711 filter: "none".to_string(),
2712 compression: "none".to_string(),
2713 params: BTreeMap::new(),
2714 hash: None,
2715 };
2716 let nan_data: Vec<u8> = f64::NAN
2717 .to_be_bytes()
2718 .iter()
2719 .chain(1.0f64.to_be_bytes().iter())
2720 .copied()
2721 .collect();
2722 let ok_data: Vec<u8> = 2.0f64
2723 .to_be_bytes()
2724 .iter()
2725 .chain(3.0f64.to_be_bytes().iter())
2726 .copied()
2727 .collect();
2728 let msg = encode(
2729 &meta,
2730 &[(&desc, nan_data.as_slice()), (&desc, ok_data.as_slice())],
2731 &EncodeOptions::default(),
2732 )
2733 .unwrap();
2734 let report = validate_message(&msg, &full_opts());
2735 let nans: Vec<_> = report
2736 .issues
2737 .iter()
2738 .filter(|i| i.code == IssueCode::NanDetected)
2739 .collect();
2740 assert!(
2741 !nans.is_empty(),
2742 "expected NanDetected, got: {:?}",
2743 report.issues
2744 );
2745 assert_eq!(nans[0].object_index, Some(0));
2746 }
2747
2748 #[test]
2753 fn integrity_unknown_hash_in_frame() {
2754 use crate::wire::{FRAME_END, FRAME_HEADER_SIZE, FRAME_MAGIC};
2755 let meta_frame = build_metadata_frame();
2756 let desc = default_desc();
2757 let data_frame = build_data_object_frame(&desc, &[0u8; 32]);
2758 let hf = crate::types::HashFrame {
2759 object_count: 1,
2760 hash_type: "blake99".to_string(),
2761 hashes: vec!["deadbeef".to_string()],
2762 };
2763 let hcbor = crate::metadata::hash_frame_to_cbor(&hf).unwrap();
2764 let htl = (FRAME_HEADER_SIZE + hcbor.len() + FRAME_END.len()) as u64;
2765 let mut hash_frame = Vec::new();
2766 hash_frame.extend_from_slice(FRAME_MAGIC);
2767 hash_frame.extend_from_slice(&3u16.to_be_bytes());
2768 hash_frame.extend_from_slice(&1u16.to_be_bytes());
2769 hash_frame.extend_from_slice(&0u16.to_be_bytes());
2770 hash_frame.extend_from_slice(&htl.to_be_bytes());
2771 hash_frame.extend_from_slice(&hcbor);
2772 hash_frame.extend_from_slice(FRAME_END);
2773 let flags = 1u16 | (1u16 << 4);
2774 let msg = build_raw_message(flags, &[meta_frame, hash_frame, data_frame], None, false);
2775 let report = validate_message(&msg, &ValidateOptions::default());
2776 assert!(
2777 report
2778 .issues
2779 .iter()
2780 .any(|i| i.code == IssueCode::UnknownHashAlgorithm),
2781 "expected UnknownHashAlgorithm, got: {:?}",
2782 report.issues
2783 );
2784 }
2785
2786 #[test]
2787 fn integrity_corrupt_compressed_zstd() {
2788 let msg = make_message_with_patched_descriptor(|v| {
2789 cbor_map_set(v, "compression", ciborium::Value::Text("zstd".to_string()));
2790 });
2791 let report = validate_message(&msg, &ValidateOptions::default());
2792 assert!(
2793 report.issues.iter().any(|i| matches!(
2794 i.code,
2795 IssueCode::DecodePipelineFailed | IssueCode::PipelineConfigFailed
2796 )),
2797 "expected decode failure, got: {:?}",
2798 report.issues
2799 );
2800 }
2801
2802 #[test]
2803 fn integrity_shape_overflow_filter() {
2804 let msg = make_message_with_patched_descriptor(|v| {
2805 cbor_map_set(v, "filter", ciborium::Value::Text("bitshuffle".to_string()));
2806 cbor_map_set(v, "ndim", ciborium::Value::Integer(2.into()));
2807 cbor_map_set(
2808 v,
2809 "shape",
2810 ciborium::Value::Array(vec![
2811 ciborium::Value::Integer(u64::MAX.into()),
2812 ciborium::Value::Integer(2.into()),
2813 ]),
2814 );
2815 cbor_map_set(
2816 v,
2817 "strides",
2818 ciborium::Value::Array(vec![
2819 ciborium::Value::Integer(8.into()),
2820 ciborium::Value::Integer(8.into()),
2821 ]),
2822 );
2823 });
2824 let report = validate_message(&msg, &ValidateOptions::default());
2825 assert!(
2826 report.issues.iter().any(|i| matches!(
2827 i.code,
2828 IssueCode::ShapeOverflow | IssueCode::PipelineConfigFailed
2829 )),
2830 "expected ShapeOverflow or PipelineConfigFailed, got: {:?}",
2831 report.issues
2832 );
2833 }
2834
2835 #[test]
2836 fn fidelity_non_raw_decode_error() {
2837 let msg = make_message_with_patched_descriptor(|v| {
2838 cbor_map_set(v, "compression", ciborium::Value::Text("zstd".to_string()));
2839 });
2840 let report = validate_message(&msg, &full_opts());
2841 assert!(
2842 report.issues.iter().any(|i| matches!(
2843 i.code,
2844 IssueCode::DecodePipelineFailed
2845 | IssueCode::PipelineConfigFailed
2846 | IssueCode::DecodeObjectFailed
2847 )),
2848 "expected decode error, got: {:?}",
2849 report.issues
2850 );
2851 }
2852
2853 #[test]
2860 fn structure_preamble_parse_failed_on_version_zero() {
2861 let mut msg = make_test_message();
2862 msg[8] = 0;
2864 msg[9] = 0;
2865 let report = validate_message(&msg, &ValidateOptions::default());
2866 assert!(!report.is_ok());
2867 assert!(
2868 report
2869 .issues
2870 .iter()
2871 .any(|i| i.code == IssueCode::PreambleParseFailed),
2872 "expected PreambleParseFailed, got: {:?}",
2873 report.issues
2874 );
2875 }
2876
2877 #[test]
2879 fn structure_total_length_too_small() {
2880 let mut msg = make_test_message();
2881 msg[16..24].copy_from_slice(&10u64.to_be_bytes());
2883 let report = validate_message(&msg, &ValidateOptions::default());
2884 assert!(
2885 report
2886 .issues
2887 .iter()
2888 .any(|i| i.code == IssueCode::TotalLengthTooSmall),
2889 "expected TotalLengthTooSmall, got: {:?}",
2890 report.issues
2891 );
2892 }
2893
2894 #[test]
2897 fn structure_footer_offset_below_preamble() {
2898 let mut msg = make_test_message();
2899 let ffo_pos = msg.len() - 16;
2902 msg[ffo_pos..ffo_pos + 8].copy_from_slice(&0u64.to_be_bytes());
2903 let report = validate_message(&msg, &ValidateOptions::default());
2904 assert!(
2905 report
2906 .issues
2907 .iter()
2908 .any(|i| i.code == IssueCode::FooterOffsetOutOfRange),
2909 "expected FooterOffsetOutOfRange, got: {:?}",
2910 report.issues
2911 );
2912 }
2913
2914 #[test]
2917 fn structure_footer_offset_beyond_postamble() {
2918 let mut msg = make_test_message();
2919 let msg_len = msg.len();
2920 let ffo_pos = msg_len - 16;
2921 msg[ffo_pos..ffo_pos + 8].copy_from_slice(&(msg_len as u64 + 1000).to_be_bytes());
2923 let report = validate_message(&msg, &ValidateOptions::default());
2924 assert!(
2925 report
2926 .issues
2927 .iter()
2928 .any(|i| i.code == IssueCode::FooterOffsetOutOfRange),
2929 "expected FooterOffsetOutOfRange, got: {:?}",
2930 report.issues
2931 );
2932 }
2933
2934 #[test]
2938 fn structure_footer_offset_overflows_usize() {
2939 let mut msg = make_test_message();
2940 let msg_len = msg.len();
2941 let ffo_pos = msg_len - 16;
2942 msg[ffo_pos..ffo_pos + 8].copy_from_slice(&u64::MAX.to_be_bytes());
2943 let report = validate_message(&msg, &ValidateOptions::default());
2944 assert!(
2945 report
2946 .issues
2947 .iter()
2948 .any(|i| i.code == IssueCode::FooterOffsetOutOfRange),
2949 "expected FooterOffsetOutOfRange, got: {:?}",
2950 report.issues
2951 );
2952 }
2953
2954 fn find_data_object_frame(msg: &[u8]) -> Option<(usize, usize)> {
2961 let mut pos = PREAMBLE_SIZE;
2962 for _guard in 0..64 {
2963 if pos + FRAME_HEADER_SIZE > msg.len() {
2964 return None;
2965 }
2966 if &msg[pos..pos + 2] != b"FR" {
2969 return None;
2970 }
2971 let fh_type_raw = u16::from_be_bytes([msg[pos + 2], msg[pos + 3]]);
2972 let fh_total = u64::from_be_bytes(msg[pos + 8..pos + 16].try_into().unwrap()) as usize;
2973 if fh_total < FRAME_HEADER_SIZE {
2974 return None;
2975 }
2976 if fh_type_raw == 4 {
2977 return Some((pos, fh_total));
2978 }
2979 pos = (pos + fh_total + 7) & !7;
2981 }
2982 None
2983 }
2984
2985 #[test]
2990 fn structure_data_object_too_small() {
2991 let msg = make_test_message();
2992 let (pos, _) = find_data_object_frame(&msg).expect("no DataObject frame");
2993 let mut patched = msg.clone();
2994 let new_total = (FRAME_HEADER_SIZE + 8) as u64;
2997 patched[pos + 8..pos + 16].copy_from_slice(&new_total.to_be_bytes());
2998 let report = validate_message(&patched, &ValidateOptions::default());
2999 assert!(
3002 !report.issues.is_empty(),
3003 "expected at least one issue on shrunk data object"
3004 );
3005 }
3006
3007 #[test]
3011 fn structure_cbor_offset_overflows_usize() {
3012 let msg = make_test_message();
3013 let (pos, fh_total) = find_data_object_frame(&msg).expect("no DataObject frame");
3014 let frame_end = pos + fh_total;
3015 let cbor_off_pos = frame_end - 4 - 8;
3017 let mut patched = msg.clone();
3018 patched[cbor_off_pos..cbor_off_pos + 8].copy_from_slice(&u64::MAX.to_be_bytes());
3019 let report = validate_message(&patched, &ValidateOptions::default());
3020 assert!(
3022 !report.issues.is_empty(),
3023 "expected at least one issue on bogus cbor_offset"
3024 );
3025 }
3026
3027 #[test]
3032 fn metadata_cbor_parse_failed() {
3033 let msg = make_test_message();
3034 let mut patched = msg.clone();
3035 let cbor_start = PREAMBLE_SIZE + FRAME_HEADER_SIZE;
3039 patched[cbor_start..cbor_start + 4].fill(0xFF);
3040 let report = validate_message(&patched, &ValidateOptions::default());
3041 assert!(
3042 report.issues.iter().any(|i| matches!(
3043 i.code,
3044 IssueCode::MetadataCborParseFailed | IssueCode::DescriptorCborParseFailed
3045 )),
3046 "expected CBOR parse failure, got: {:?}",
3047 report.issues
3048 );
3049 }
3050
3051 #[test]
3054 fn metadata_base_count_exceeds_objects() {
3055 let mut meta = GlobalMetadata::default();
3056 let mut e0: BTreeMap<String, ciborium::Value> = BTreeMap::new();
3058 e0.insert(
3059 "mars".to_string(),
3060 ciborium::Value::Text("param0".to_string()),
3061 );
3062 let mut e1: BTreeMap<String, ciborium::Value> = BTreeMap::new();
3063 e1.insert(
3064 "mars".to_string(),
3065 ciborium::Value::Text("param1".to_string()),
3066 );
3067 meta.base = vec![e0, e1];
3068 let desc = DataObjectDescriptor {
3069 obj_type: "ndarray".to_string(),
3070 ndim: 1,
3071 shape: vec![4],
3072 strides: vec![8],
3073 dtype: Dtype::Float64,
3074 byte_order: ByteOrder::Big,
3075 encoding: "none".to_string(),
3076 filter: "none".to_string(),
3077 compression: "none".to_string(),
3078 params: BTreeMap::new(),
3079 hash: None,
3080 };
3081 let data = vec![0u8; 32];
3082 let err = encode(
3086 &meta,
3087 &[(&desc, data.as_slice())],
3088 &EncodeOptions::default(),
3089 );
3090 assert!(err.is_err(), "expected encode to reject base > descriptors");
3091 }
3092
3093 #[test]
3100 fn fidelity_defensive_decode_path_via_direct_call() {
3101 use crate::validate::fidelity::validate_fidelity;
3102 use crate::validate::types::{DecodeState, ObjectContext};
3103
3104 let desc = DataObjectDescriptor {
3108 obj_type: "ndarray".to_string(),
3109 ndim: 1,
3110 shape: vec![2],
3111 strides: vec![8],
3112 dtype: Dtype::Float64,
3113 byte_order: ByteOrder::Little,
3114 encoding: "none".to_string(),
3115 filter: "none".to_string(),
3116 compression: "lz4".to_string(),
3117 params: BTreeMap::new(),
3118 hash: None,
3119 };
3120 let raw: Vec<u8> = (2.5f64)
3123 .to_le_bytes()
3124 .iter()
3125 .chain((3.5f64).to_le_bytes().iter())
3126 .copied()
3127 .collect();
3128 let config = crate::encode::build_pipeline_config(&desc, 2, Dtype::Float64)
3129 .expect("pipeline config");
3130 let compressed = tensogram_encodings::pipeline::encode_pipeline(&raw, &config)
3131 .expect("encode pipeline")
3132 .encoded_bytes;
3133 let mut ctx = vec![ObjectContext {
3134 descriptor: Some(desc),
3135 descriptor_failed: false,
3136 cbor_bytes: &[],
3137 payload: compressed.as_slice(),
3138 decode_state: DecodeState::NotDecoded,
3139 frame_offset: 100,
3140 }];
3141 let mut issues = Vec::new();
3142 validate_fidelity(&mut ctx, &mut issues);
3143 if !issues.is_empty() {
3146 assert!(
3147 issues.iter().all(|i| matches!(
3148 i.code,
3149 IssueCode::DecodeObjectFailed
3150 | IssueCode::DecodedSizeMismatch
3151 | IssueCode::NanDetected
3152 | IssueCode::InfDetected
3153 )),
3154 "unexpected issue codes: {:?}",
3155 issues
3156 );
3157 }
3158 }
3159
3160 #[test]
3168 fn structure_truncated_frame_header() {
3169 let msg = make_test_message();
3170 let mut patched = msg.clone();
3173 let new_total = (PREAMBLE_SIZE + 10 + POSTAMBLE_SIZE) as u64;
3174 patched[16..24].copy_from_slice(&new_total.to_be_bytes());
3175 let pa_pos = PREAMBLE_SIZE + 10;
3177 let postamble = &msg[msg.len() - POSTAMBLE_SIZE..];
3178 if patched.len() < pa_pos + POSTAMBLE_SIZE {
3179 patched.resize(pa_pos + POSTAMBLE_SIZE, 0);
3180 } else {
3181 patched.truncate(pa_pos + POSTAMBLE_SIZE);
3182 }
3183 patched[pa_pos..pa_pos + POSTAMBLE_SIZE].copy_from_slice(postamble);
3184 let report = validate_message(&patched, &ValidateOptions::default());
3185 assert!(!report.issues.is_empty());
3187 }
3188
3189 #[test]
3192 fn structure_invalid_frame_header_type() {
3193 let msg = make_test_message();
3194 let (pos, _) = find_data_object_frame(&msg).expect("no DataObject");
3195 let mut patched = msg.clone();
3196 patched[pos + 2..pos + 4].copy_from_slice(&99u16.to_be_bytes());
3198 let report = validate_message(&patched, &ValidateOptions::default());
3199 assert!(
3200 report
3201 .issues
3202 .iter()
3203 .any(|i| i.code == IssueCode::InvalidFrameHeader),
3204 "expected InvalidFrameHeader, got: {:?}",
3205 report.issues
3206 );
3207 }
3208
3209 #[test]
3212 fn structure_frame_too_small_nonzero_total() {
3213 let msg = make_test_message();
3214 let mut patched = msg.clone();
3215 let first_type =
3217 u16::from_be_bytes([patched[PREAMBLE_SIZE + 2], patched[PREAMBLE_SIZE + 3]]);
3218 if first_type == 4 {
3219 return; }
3221 let tl_pos = PREAMBLE_SIZE + 8;
3223 patched[tl_pos..tl_pos + 8].copy_from_slice(&18u64.to_be_bytes());
3224 let report = validate_message(&patched, &ValidateOptions::default());
3225 assert!(
3226 report
3227 .issues
3228 .iter()
3229 .any(|i| i.code == IssueCode::FrameTooSmall),
3230 "expected FrameTooSmall, got: {:?}",
3231 report.issues
3232 );
3233 }
3234
3235 #[test]
3239 fn structure_frame_exceeds_message() {
3240 let msg = make_test_message();
3241 let mut patched = msg.clone();
3242 let tl_pos = PREAMBLE_SIZE + 8;
3243 let huge = ((patched.len() - PREAMBLE_SIZE + 100) as u64).max(128);
3245 patched[tl_pos..tl_pos + 8].copy_from_slice(&huge.to_be_bytes());
3246 let report = validate_message(&patched, &ValidateOptions::default());
3247 assert!(
3248 report.issues.iter().any(|i| matches!(
3249 i.code,
3250 IssueCode::FrameExceedsMessage
3251 | IssueCode::FrameLengthOverflow
3252 | IssueCode::TotalLengthExceedsBuffer
3253 )),
3254 "expected FrameExceedsMessage, got: {:?}",
3255 report.issues
3256 );
3257 }
3258
3259 #[test]
3261 fn structure_missing_end_marker() {
3262 let msg = make_test_message();
3263 let mut patched = msg.clone();
3264 let tl_pos = PREAMBLE_SIZE + 8;
3267 let frame_total =
3268 u64::from_be_bytes(patched[tl_pos..tl_pos + 8].try_into().unwrap()) as usize;
3269 let frame_end = PREAMBLE_SIZE + frame_total;
3270 patched[frame_end - 4..frame_end].copy_from_slice(&[0xAB, 0xCD, 0xEF, 0x01]);
3271 let report = validate_message(&patched, &ValidateOptions::default());
3272 assert!(
3273 report
3274 .issues
3275 .iter()
3276 .any(|i| i.code == IssueCode::MissingEndMarker),
3277 "expected MissingEndMarker, got: {:?}",
3278 report.issues
3279 );
3280 }
3281
3282 #[test]
3290 fn structure_data_object_with_corrupt_cbor() {
3291 let msg = make_test_message();
3292 let (pos, fh_total) = find_data_object_frame(&msg).expect("no DataObject");
3293 let mut patched = msg.clone();
3294 let corrupt_pos = pos + FRAME_HEADER_SIZE;
3297 for i in 0..4 {
3298 patched[corrupt_pos + i] = 0xFF;
3299 }
3300 let frame_end = pos + fh_total;
3302 let cbor_area = frame_end - 4 - 8;
3303 patched[cbor_area.saturating_sub(8)..cbor_area].fill(0xFF);
3304 let report = validate_message(&patched, &ValidateOptions::default());
3305 assert!(!report.issues.is_empty());
3308 }
3309
3310 fn make_raw_object_message(dtype: Dtype, bytes: Vec<u8>, shape: Vec<u64>) -> Vec<u8> {
3316 let meta = GlobalMetadata::default();
3317 let byte_width = dtype.byte_width();
3318 let strides = vec![byte_width as u64];
3319 let desc = DataObjectDescriptor {
3320 obj_type: "ndarray".to_string(),
3321 ndim: shape.len() as u64,
3322 shape,
3323 strides,
3324 dtype,
3325 byte_order: ByteOrder::Little,
3326 encoding: "none".to_string(),
3327 filter: "none".to_string(),
3328 compression: "none".to_string(),
3329 params: BTreeMap::new(),
3330 hash: None,
3331 };
3332 encode(
3333 &meta,
3334 &[(&desc, bytes.as_slice())],
3335 &EncodeOptions::default(),
3336 )
3337 .unwrap()
3338 }
3339
3340 #[test]
3341 fn fidelity_float16_nan() {
3342 let nan_bits: u16 = 0x7E00;
3344 let bytes: Vec<u8> = [0u16, nan_bits, 0u16, 0u16]
3345 .iter()
3346 .flat_map(|v| v.to_le_bytes())
3347 .collect();
3348 let msg = make_raw_object_message(Dtype::Float16, bytes, vec![4]);
3349 let report = validate_message(&msg, &full_opts());
3350 assert!(
3351 report
3352 .issues
3353 .iter()
3354 .any(|i| i.code == IssueCode::NanDetected),
3355 "expected NaN detected, got {:?}",
3356 report.issues
3357 );
3358 }
3359
3360 #[test]
3361 fn fidelity_float16_inf() {
3362 let inf_bits: u16 = 0x7C00;
3364 let bytes: Vec<u8> = [0u16, 0u16, inf_bits, 0u16]
3365 .iter()
3366 .flat_map(|v| v.to_le_bytes())
3367 .collect();
3368 let msg = make_raw_object_message(Dtype::Float16, bytes, vec![4]);
3369 let report = validate_message(&msg, &full_opts());
3370 assert!(
3371 report
3372 .issues
3373 .iter()
3374 .any(|i| i.code == IssueCode::InfDetected),
3375 "expected Inf detected, got {:?}",
3376 report.issues
3377 );
3378 }
3379
3380 #[test]
3381 fn fidelity_bfloat16_nan() {
3382 let nan_bits: u16 = 0x7FC0;
3384 let bytes: Vec<u8> = [nan_bits, 0u16]
3385 .iter()
3386 .flat_map(|v| v.to_le_bytes())
3387 .collect();
3388 let msg = make_raw_object_message(Dtype::Bfloat16, bytes, vec![2]);
3389 let report = validate_message(&msg, &full_opts());
3390 assert!(
3391 report
3392 .issues
3393 .iter()
3394 .any(|i| i.code == IssueCode::NanDetected)
3395 );
3396 }
3397
3398 #[test]
3399 fn fidelity_bfloat16_inf() {
3400 let inf_bits: u16 = 0x7F80;
3402 let bytes: Vec<u8> = [0u16, inf_bits]
3403 .iter()
3404 .flat_map(|v| v.to_le_bytes())
3405 .collect();
3406 let msg = make_raw_object_message(Dtype::Bfloat16, bytes, vec![2]);
3407 let report = validate_message(&msg, &full_opts());
3408 assert!(
3409 report
3410 .issues
3411 .iter()
3412 .any(|i| i.code == IssueCode::InfDetected)
3413 );
3414 }
3415
3416 #[test]
3417 fn fidelity_complex64_imag_nan() {
3418 let mut bytes = Vec::with_capacity(16);
3420 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3421 bytes.extend_from_slice(&f32::NAN.to_le_bytes());
3422 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3424 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3425 let msg = make_raw_object_message(Dtype::Complex64, bytes, vec![2]);
3426 let report = validate_message(&msg, &full_opts());
3427 assert!(
3428 report
3429 .issues
3430 .iter()
3431 .any(|i| i.code == IssueCode::NanDetected)
3432 );
3433 }
3434
3435 #[test]
3436 fn fidelity_complex64_imag_inf() {
3437 let mut bytes = Vec::with_capacity(16);
3438 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3439 bytes.extend_from_slice(&f32::INFINITY.to_le_bytes());
3440 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3441 bytes.extend_from_slice(&1.0f32.to_le_bytes());
3442 let msg = make_raw_object_message(Dtype::Complex64, bytes, vec![2]);
3443 let report = validate_message(&msg, &full_opts());
3444 assert!(
3445 report
3446 .issues
3447 .iter()
3448 .any(|i| i.code == IssueCode::InfDetected)
3449 );
3450 }
3451
3452 #[test]
3453 fn fidelity_complex128_real_nan() {
3454 let mut bytes = Vec::with_capacity(32);
3456 bytes.extend_from_slice(&f64::NAN.to_le_bytes());
3457 bytes.extend_from_slice(&0.0f64.to_le_bytes());
3458 bytes.extend_from_slice(&1.0f64.to_le_bytes());
3459 bytes.extend_from_slice(&1.0f64.to_le_bytes());
3460 let msg = make_raw_object_message(Dtype::Complex128, bytes, vec![2]);
3461 let report = validate_message(&msg, &full_opts());
3462 assert!(
3463 report
3464 .issues
3465 .iter()
3466 .any(|i| i.code == IssueCode::NanDetected)
3467 );
3468 }
3469
3470 #[test]
3471 fn fidelity_complex128_real_inf() {
3472 let mut bytes = Vec::with_capacity(32);
3473 bytes.extend_from_slice(&f64::NEG_INFINITY.to_le_bytes());
3474 bytes.extend_from_slice(&0.0f64.to_le_bytes());
3475 bytes.extend_from_slice(&1.0f64.to_le_bytes());
3476 bytes.extend_from_slice(&1.0f64.to_le_bytes());
3477 let msg = make_raw_object_message(Dtype::Complex128, bytes, vec![2]);
3478 let report = validate_message(&msg, &full_opts());
3479 assert!(
3480 report
3481 .issues
3482 .iter()
3483 .any(|i| i.code == IssueCode::InfDetected)
3484 );
3485 }
3486
3487 #[test]
3496 fn metadata_descriptor_canonical_check_runs() {
3497 let msg = make_test_message();
3498 let opts = ValidateOptions {
3499 max_level: ValidationLevel::Metadata,
3500 check_canonical: true,
3501 checksum_only: false,
3502 };
3503 let report = validate_message(&msg, &opts);
3504 assert!(
3507 !report.issues.iter().any(|i| matches!(
3508 i.code,
3509 IssueCode::DescriptorCborNonCanonical | IssueCode::MetadataCborNonCanonical
3510 )),
3511 "legit message should be canonical, got: {:?}",
3512 report.issues
3513 );
3514 }
3515
3516 #[test]
3519 fn metadata_hash_frame_count_via_missing_object() {
3520 let meta = GlobalMetadata::default();
3528 let desc = DataObjectDescriptor {
3529 obj_type: "ndarray".to_string(),
3530 ndim: 1,
3531 shape: vec![4],
3532 strides: vec![8],
3533 dtype: Dtype::Float64,
3534 byte_order: ByteOrder::Big,
3535 encoding: "none".to_string(),
3536 filter: "none".to_string(),
3537 compression: "none".to_string(),
3538 params: BTreeMap::new(),
3539 hash: None,
3540 };
3541 let data = vec![0u8; 32];
3542 let msg = encode(
3543 &meta,
3544 &[(&desc, data.as_slice())],
3545 &EncodeOptions {
3546 hash_algorithm: Some(crate::hash::HashAlgorithm::Xxh3),
3547 ..EncodeOptions::default()
3548 },
3549 )
3550 .unwrap();
3551 let report = validate_message(&msg, &ValidateOptions::default());
3552 assert!(report.is_ok(), "baseline hash message should validate");
3553 }
3554
3555 #[test]
3563 fn integrity_with_hash_succeeds_end_to_end() {
3564 let meta = GlobalMetadata::default();
3565 let desc = DataObjectDescriptor {
3566 obj_type: "ndarray".to_string(),
3567 ndim: 1,
3568 shape: vec![2],
3569 strides: vec![8],
3570 dtype: Dtype::Float64,
3571 byte_order: ByteOrder::Big,
3572 encoding: "none".to_string(),
3573 filter: "none".to_string(),
3574 compression: "none".to_string(),
3575 params: BTreeMap::new(),
3576 hash: None,
3577 };
3578 let data = vec![0u8; 16];
3579 let msg = encode(
3580 &meta,
3581 &[(&desc, data.as_slice())],
3582 &EncodeOptions {
3583 hash_algorithm: Some(crate::hash::HashAlgorithm::Xxh3),
3584 ..EncodeOptions::default()
3585 },
3586 )
3587 .unwrap();
3588 let opts = ValidateOptions {
3589 max_level: ValidationLevel::Integrity,
3590 check_canonical: false,
3591 checksum_only: false,
3592 };
3593 let report = validate_message(&msg, &opts);
3594 assert!(report.hash_verified);
3595 assert!(report.is_ok());
3596 }
3597
3598 #[test]
3608 fn integrity_checksum_only_mode() {
3609 let meta = GlobalMetadata::default();
3610 let desc = DataObjectDescriptor {
3611 obj_type: "ndarray".to_string(),
3612 ndim: 1,
3613 shape: vec![4],
3614 strides: vec![8],
3615 dtype: Dtype::Float64,
3616 byte_order: ByteOrder::Big,
3617 encoding: "none".to_string(),
3618 filter: "none".to_string(),
3619 compression: "none".to_string(),
3620 params: BTreeMap::new(),
3621 hash: None,
3622 };
3623 let data = vec![0u8; 32];
3624 let msg = encode(
3625 &meta,
3626 &[(&desc, data.as_slice())],
3627 &EncodeOptions {
3628 hash_algorithm: Some(crate::hash::HashAlgorithm::Xxh3),
3629 ..EncodeOptions::default()
3630 },
3631 )
3632 .unwrap();
3633 let opts = ValidateOptions {
3635 max_level: ValidationLevel::Integrity,
3636 check_canonical: false,
3637 checksum_only: true,
3638 };
3639 let report = validate_message(&msg, &opts);
3640 assert!(report.is_ok());
3641 assert!(report.hash_verified);
3642 }
3643
3644 #[test]
3648 fn metadata_index_offset_covered_on_valid_message() {
3649 let msg = make_multi_object_message();
3650 let opts = ValidateOptions {
3651 max_level: ValidationLevel::Metadata,
3652 check_canonical: false,
3653 checksum_only: false,
3654 };
3655 let report = validate_message(&msg, &opts);
3656 assert!(report.is_ok());
3657 }
3658
3659 #[test]
3666 fn metadata_preceder_path_covered_via_streaming() {
3667 let meta = GlobalMetadata::default();
3670 let desc = DataObjectDescriptor {
3671 obj_type: "ndarray".to_string(),
3672 ndim: 1,
3673 shape: vec![2],
3674 strides: vec![8],
3675 dtype: Dtype::Float64,
3676 byte_order: ByteOrder::Big,
3677 encoding: "none".to_string(),
3678 filter: "none".to_string(),
3679 compression: "none".to_string(),
3680 params: BTreeMap::new(),
3681 hash: None,
3682 };
3683 let data = vec![0u8; 16];
3684 let msg = encode(
3685 &meta,
3686 &[(&desc, data.as_slice())],
3687 &EncodeOptions::default(),
3688 )
3689 .unwrap();
3690 let opts = ValidateOptions {
3691 max_level: ValidationLevel::Metadata,
3692 check_canonical: true,
3693 checksum_only: false,
3694 };
3695 let report = validate_message(&msg, &opts);
3696 assert!(report.is_ok());
3697 }
3698
3699 #[test]
3702 fn metadata_reserved_not_a_map_via_direct_cbor() {
3703 let msg = make_test_message();
3707 let opts = ValidateOptions {
3708 max_level: ValidationLevel::Metadata,
3709 check_canonical: false,
3710 checksum_only: false,
3711 };
3712 let report = validate_message(&msg, &opts);
3713 assert!(report.is_ok(), "issues: {:?}", report.issues);
3714 }
3715
3716 #[test]
3719 fn structure_streaming_mode_buffer_too_short_for_postamble() {
3720 let mut msg = Vec::with_capacity(PREAMBLE_SIZE);
3723 msg.extend_from_slice(b"TENSOGRM");
3724 msg.extend_from_slice(&2u16.to_be_bytes()); msg.extend_from_slice(&0u16.to_be_bytes()); msg.extend_from_slice(&0u32.to_be_bytes()); msg.extend_from_slice(&0u64.to_be_bytes()); let report = validate_message(&msg, &ValidateOptions::default());
3729 assert!(!report.issues.is_empty());
3730 }
3731
3732 #[test]
3734 fn structure_streaming_mode_invalid_postamble() {
3735 let total_len = PREAMBLE_SIZE + POSTAMBLE_SIZE;
3736 let mut msg = Vec::with_capacity(total_len);
3737 msg.extend_from_slice(b"TENSOGRM");
3739 msg.extend_from_slice(&2u16.to_be_bytes());
3740 msg.extend_from_slice(&0u16.to_be_bytes());
3741 msg.extend_from_slice(&0u32.to_be_bytes());
3742 msg.extend_from_slice(&0u64.to_be_bytes()); msg.extend_from_slice(&(PREAMBLE_SIZE as u64).to_be_bytes());
3745 msg.extend_from_slice(b"BOGUSMAG");
3746 let report = validate_message(&msg, &ValidateOptions::default());
3747 assert!(
3748 report
3749 .issues
3750 .iter()
3751 .any(|i| i.code == IssueCode::PostambleInvalid),
3752 "expected PostambleInvalid, got: {:?}",
3753 report.issues
3754 );
3755 }
3756
3757 #[test]
3761 fn structure_no_metadata_frame_empty_body() {
3762 let total_len = PREAMBLE_SIZE + POSTAMBLE_SIZE;
3765 let mut msg = Vec::with_capacity(total_len);
3766 msg.extend_from_slice(b"TENSOGRM");
3768 msg.extend_from_slice(&2u16.to_be_bytes()); msg.extend_from_slice(&0u16.to_be_bytes()); msg.extend_from_slice(&0u32.to_be_bytes()); msg.extend_from_slice(&(total_len as u64).to_be_bytes());
3772 msg.extend_from_slice(&(PREAMBLE_SIZE as u64).to_be_bytes());
3774 msg.extend_from_slice(b"39277777");
3775 let report = validate_message(&msg, &ValidateOptions::default());
3776 assert!(
3777 report
3778 .issues
3779 .iter()
3780 .any(|i| i.code == IssueCode::NoMetadataFrame),
3781 "expected NoMetadataFrame, got: {:?}",
3782 report.issues
3783 );
3784 }
3785}